@vercel/agent-eval 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -3
- package/dist/cli.js +4 -0
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/agents/claude-code.d.ts.map +1 -1
- package/dist/lib/agents/claude-code.js +4 -7
- package/dist/lib/agents/claude-code.js.map +1 -1
- package/dist/lib/agents/codex.d.ts.map +1 -1
- package/dist/lib/agents/codex.js +4 -7
- package/dist/lib/agents/codex.js.map +1 -1
- package/dist/lib/agents/index.d.ts.map +1 -1
- package/dist/lib/agents/index.js +2 -0
- package/dist/lib/agents/index.js.map +1 -1
- package/dist/lib/agents/opencode.d.ts +11 -0
- package/dist/lib/agents/opencode.d.ts.map +1 -0
- package/dist/lib/agents/opencode.js +221 -0
- package/dist/lib/agents/opencode.js.map +1 -0
- package/dist/lib/agents/shared.d.ts +7 -3
- package/dist/lib/agents/shared.d.ts.map +1 -1
- package/dist/lib/agents/shared.js.map +1 -1
- package/dist/lib/agents/types.d.ts +3 -1
- package/dist/lib/agents/types.d.ts.map +1 -1
- package/dist/lib/config.d.ts +1 -0
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/config.js +4 -0
- package/dist/lib/config.js.map +1 -1
- package/dist/lib/docker-sandbox.d.ts +91 -0
- package/dist/lib/docker-sandbox.d.ts.map +1 -0
- package/dist/lib/docker-sandbox.js +324 -0
- package/dist/lib/docker-sandbox.js.map +1 -0
- package/dist/lib/runner.d.ts +1 -0
- package/dist/lib/runner.d.ts.map +1 -1
- package/dist/lib/runner.js +41 -8
- package/dist/lib/runner.js.map +1 -1
- package/dist/lib/sandbox.d.ts +53 -2
- package/dist/lib/sandbox.d.ts.map +1 -1
- package/dist/lib/sandbox.js +77 -1
- package/dist/lib/sandbox.js.map +1 -1
- package/dist/lib/types.d.ts +8 -1
- package/dist/lib/types.d.ts.map +1 -1
- package/dist/lib/types.js.map +1 -1
- package/package.json +9 -1
package/dist/lib/runner.js
CHANGED
|
@@ -51,15 +51,47 @@ export async function runExperiment(options) {
|
|
|
51
51
|
};
|
|
52
52
|
}
|
|
53
53
|
log(createProgressDisplay(fixture.name, runIndex + 1, config.runs));
|
|
54
|
-
const
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
54
|
+
const timeoutMs = config.timeout * 1000;
|
|
55
|
+
const startTime = Date.now();
|
|
56
|
+
// Create per-attempt controller for timeout cleanup
|
|
57
|
+
const attemptController = new AbortController();
|
|
58
|
+
// Propagate earlyExit abort to this attempt's controller
|
|
59
|
+
if (config.earlyExit) {
|
|
60
|
+
controller.signal.addEventListener('abort', () => attemptController.abort(), { once: true });
|
|
61
|
+
}
|
|
62
|
+
let timeoutId;
|
|
63
|
+
const agentResult = await Promise.race([
|
|
64
|
+
agent.run(fixture.path, {
|
|
65
|
+
prompt: fixture.prompt,
|
|
66
|
+
model: config.model,
|
|
67
|
+
timeout: timeoutMs,
|
|
68
|
+
apiKey,
|
|
69
|
+
setup: config.setup,
|
|
70
|
+
scripts: config.scripts,
|
|
71
|
+
signal: attemptController.signal,
|
|
72
|
+
sandbox: config.sandbox,
|
|
73
|
+
}),
|
|
74
|
+
new Promise((_, reject) => {
|
|
75
|
+
timeoutId = setTimeout(() => {
|
|
76
|
+
attemptController.abort(); // Signal agent to clean up sandbox
|
|
77
|
+
reject(new Error(`Eval timed out after ${config.timeout}s`));
|
|
78
|
+
}, timeoutMs);
|
|
79
|
+
}),
|
|
80
|
+
]).catch((error) => {
|
|
81
|
+
// Convert timeout error to AgentResult format
|
|
82
|
+
if (error instanceof Error && error.message.includes('timed out')) {
|
|
83
|
+
return {
|
|
84
|
+
success: false,
|
|
85
|
+
output: '',
|
|
86
|
+
error: error.message,
|
|
87
|
+
duration: Date.now() - startTime,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
throw error;
|
|
62
91
|
});
|
|
92
|
+
// Clear timeout if agent completed before timeout
|
|
93
|
+
if (timeoutId)
|
|
94
|
+
clearTimeout(timeoutId);
|
|
63
95
|
// Check if this was aborted
|
|
64
96
|
if (agentResult.error === 'Aborted' || agentResult.error === 'Aborted before start') {
|
|
65
97
|
return {
|
|
@@ -136,6 +168,7 @@ export async function runSingleEval(fixture, options) {
|
|
|
136
168
|
apiKey: options.apiKey,
|
|
137
169
|
setup: options.setup,
|
|
138
170
|
scripts: options.scripts,
|
|
171
|
+
sandbox: options.sandbox,
|
|
139
172
|
});
|
|
140
173
|
return agentResultToEvalRunData(agentResult);
|
|
141
174
|
}
|
package/dist/lib/runner.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AASH,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,qBAAqB,GACtB,MAAM,cAAc,CAAC;AAwCtB;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAA6B;IAE7B,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAC9F,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAE7B,8BAA8B;IAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAErC,MAAM,GAAG,GAAG,CAAC,GAAW,EAAE,EAAE;QAC1B,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;aAAM,IAAI,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC,CAAC;IAEF,mDAAmD;IACnD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,eAAe,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,oCAAoC;IACpC,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,GAAG,CAAC,YAAY,QAAQ,CAAC,MAAM,gCAAgC,QAAQ,CAAC,MAAM,YAAY,MAAM,CAAC,IAAI,QAAQ,CAAC,CAAC;IAE/G,uBAAuB;IACvB,MAAM,UAAU,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACxE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QACtC,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAEvD,2CAA2C;QAC3C,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC9B,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,EAAE;iBAC5D;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,qBAAqB,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpE,MAAM,
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AASH,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,qBAAqB,GACtB,MAAM,cAAc,CAAC;AAwCtB;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAA6B;IAE7B,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAC9F,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAE7B,8BAA8B;IAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAErC,MAAM,GAAG,GAAG,CAAC,GAAW,EAAE,EAAE;QAC1B,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;aAAM,IAAI,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC,CAAC;IAEF,mDAAmD;IACnD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,eAAe,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,oCAAoC;IACpC,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,GAAG,CAAC,YAAY,QAAQ,CAAC,MAAM,gCAAgC,QAAQ,CAAC,MAAM,YAAY,MAAM,CAAC,IAAI,QAAQ,CAAC,CAAC;IAE/G,uBAAuB;IACvB,MAAM,UAAU,GAAG,KAAK,EAAE,OAAoB,EAA0B,EAAE;QACxE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QACtC,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAEvD,2CAA2C;QAC3C,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC9B,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,EAAE;iBAC5D;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,qBAAqB,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpE,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,oDAAoD;QACpD,MAAM,iBAAiB,GAAG,IAAI,eAAe,EAAE,CAAC;QAEhD,yDAAyD;QACzD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,IAAI,SAAoD,CAAC;QAEzD,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YACrC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;gBACtB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,SAAS;gBAClB,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,MAAM,EAAE,iBAAiB,CAAC,MAAM;gBAChC,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,CAAC;YACF,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/B,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC1B,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,mCAAmC;oBAC9D,MAAM,CAAC,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;gBAC/D,CAAC,EAAE,SAAS,CAAC,CAAC;YAChB,CAAC,CAAC;SACH,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACjB,8CAA8C;YAC9C,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAClE,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,EAAE;oBACV,KAAK,EAAE,KAAK,CAAC,OAAO;oBACpB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACjC,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,kDAAkD;QAClD,IAAI,SAAS;YAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QAEvC,4BAA4B;QAC5B,IAAI,WAAW,CAAC,KAAK,KAAK,SAAS,IAAI,WAAW,CAAC,KAAK,KAAK,sBAAsB,EAAE,CAAC;YACpF,OAAO;gBACL,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,QAAQ;gBACR,OAAO,EAAE;oBACP,MAAM,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE;iBACtF;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;QAEtD,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAE9E,4EAA4E;QAC5E,IAAI,MAAM,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3D,GAAG,CAAC,eAAe,OAAO,CAAC,IAAI,kBAAkB,QAAQ,GAAG,CAAC,+BAA+B,CAAC,CAAC;YAC9F,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QAED,OAAO;YACL,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,QAAQ;YACR,OAAO;SACR,CAAC;IACJ,CAAC,CAAC;IAEF,gCAAgC;IAChC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IAE5D,sDAAsD;IACtD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,MAAM,aAAa,GAAkB,EAAE,CAAC;IACxC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,cAAc,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;QAE3D,wCAAwC;QACxC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAEvD,MAAM,WAAW,GAAkB,EAAE,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAEjC,iDAAiD;YACjD,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAC7D,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;IAC/B,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAEjG,uBAAuB;IACvB,MAAM,SAAS,GAAG,WAAW,CAAC,iBAAiB,EAAE;QAC/C,UAAU;QACV,cAAc;KACf,CAAC,CAAC;IAEH,GAAG,CAAC,uBAAuB,SAAS,EAAE,CAAC,CAAC;IACxC,GAAG,CAAC,kBAAkB,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAE3C,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAoB,EACpB,OASC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,IAAI,+BAA+B,CAAC,CAAC;IAEzE,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;QAChD,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,IAAI;QAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;IAEH,OAAO,wBAAwB,CAAC,WAAW,CAAC,CAAC;AAC/C,CAAC"}
|
package/dist/lib/sandbox.d.ts
CHANGED
|
@@ -1,12 +1,29 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Sandbox integration for isolated eval execution.
|
|
3
|
+
* Supports both Vercel Sandbox and Docker backends.
|
|
3
4
|
*/
|
|
4
5
|
import { Sandbox as VercelSandbox } from '@vercel/sandbox';
|
|
5
6
|
import type { Sandbox } from './types.js';
|
|
7
|
+
import { DockerSandboxManager } from './docker-sandbox.js';
|
|
6
8
|
/**
|
|
7
9
|
* Default timeout for sandbox operations (10 minutes).
|
|
8
10
|
*/
|
|
9
11
|
export declare const DEFAULT_SANDBOX_TIMEOUT = 600000;
|
|
12
|
+
/**
|
|
13
|
+
* Supported sandbox backends.
|
|
14
|
+
*/
|
|
15
|
+
export type SandboxBackend = 'vercel' | 'docker';
|
|
16
|
+
/**
|
|
17
|
+
* Information about the resolved sandbox backend.
|
|
18
|
+
*/
|
|
19
|
+
export interface SandboxBackendInfo {
|
|
20
|
+
/** Which backend will be used */
|
|
21
|
+
backend: SandboxBackend;
|
|
22
|
+
/** How it was determined */
|
|
23
|
+
reason: 'explicit' | 'auto-detected';
|
|
24
|
+
/** Human-readable description */
|
|
25
|
+
description: string;
|
|
26
|
+
}
|
|
10
27
|
/**
|
|
11
28
|
* Files to ignore when copying to sandbox.
|
|
12
29
|
* These are build artifacts and dependencies that shouldn't be uploaded.
|
|
@@ -29,6 +46,8 @@ export interface SandboxOptions {
|
|
|
29
46
|
timeout?: number;
|
|
30
47
|
/** Runtime environment */
|
|
31
48
|
runtime?: 'node20' | 'node24';
|
|
49
|
+
/** Sandbox backend to use. 'auto' will use Vercel if token present, else Docker. @default 'auto' */
|
|
50
|
+
backend?: SandboxBackend | 'auto';
|
|
32
51
|
}
|
|
33
52
|
/**
|
|
34
53
|
* Result of running a command in the sandbox.
|
|
@@ -95,6 +114,38 @@ export declare class SandboxManager implements Sandbox {
|
|
|
95
114
|
*/
|
|
96
115
|
stop(): Promise<void>;
|
|
97
116
|
}
|
|
117
|
+
/**
|
|
118
|
+
* Resolve which sandbox backend to use based on options.
|
|
119
|
+
*
|
|
120
|
+
* Priority:
|
|
121
|
+
* 1. Explicit backend in options (if not 'auto')
|
|
122
|
+
* 2. Auto-detect: Vercel if token present, else Docker
|
|
123
|
+
*/
|
|
124
|
+
export declare function resolveBackend(options?: SandboxOptions): SandboxBackend;
|
|
125
|
+
/**
|
|
126
|
+
* Get information about the sandbox backend that will be used.
|
|
127
|
+
* Useful for displaying to users.
|
|
128
|
+
*/
|
|
129
|
+
export declare function getSandboxBackendInfo(options?: SandboxOptions): SandboxBackendInfo;
|
|
130
|
+
/**
|
|
131
|
+
* Create a sandbox using the appropriate backend.
|
|
132
|
+
*
|
|
133
|
+
* By default, uses Vercel Sandbox if VERCEL_TOKEN is present,
|
|
134
|
+
* otherwise falls back to Docker.
|
|
135
|
+
*
|
|
136
|
+
* @example
|
|
137
|
+
* ```typescript
|
|
138
|
+
* // Auto-detect backend
|
|
139
|
+
* const sandbox = await createSandbox();
|
|
140
|
+
*
|
|
141
|
+
* // Explicit Docker
|
|
142
|
+
* const sandbox = await createSandbox({ backend: 'docker' });
|
|
143
|
+
*
|
|
144
|
+
* // Explicit Vercel
|
|
145
|
+
* const sandbox = await createSandbox({ backend: 'vercel' });
|
|
146
|
+
* ```
|
|
147
|
+
*/
|
|
148
|
+
export declare function createSandbox(options?: SandboxOptions): Promise<SandboxManager | DockerSandboxManager>;
|
|
98
149
|
/**
|
|
99
150
|
* Collect files from a local directory for uploading to sandbox.
|
|
100
151
|
*/
|
|
@@ -112,5 +163,5 @@ export declare function splitTestFiles(files: SandboxFile[]): {
|
|
|
112
163
|
/**
|
|
113
164
|
* Verify that no test files exist in the sandbox.
|
|
114
165
|
*/
|
|
115
|
-
export declare function verifyNoTestFiles(sandbox: SandboxManager): Promise<void>;
|
|
166
|
+
export declare function verifyNoTestFiles(sandbox: SandboxManager | DockerSandboxManager): Promise<void>;
|
|
116
167
|
//# sourceMappingURL=sandbox.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sandbox.d.ts","sourceRoot":"","sources":["../../src/lib/sandbox.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"sandbox.d.ts","sourceRoot":"","sources":["../../src/lib/sandbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAE3D;;GAEG;AACH,eAAO,MAAM,uBAAuB,SAAS,CAAC;AAE9C;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,iCAAiC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC;IACrC,iCAAiC;IACjC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,UAU5B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,UAAuC,CAAC;AAEvE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,OAAO,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC9B,oGAAoG;IACpG,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,qBAAa,cAAe,YAAW,OAAO;IAC5C,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,iBAAiB,CAA6B;gBAE1C,OAAO,EAAE,aAAa;IAIlC;;OAEG;WACU,MAAM,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,cAAc,CAAC;IAQ1E;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACG,UAAU,CACd,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,MAAM,EAAO,EACnB,OAAO,GAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAO,GAC7C,OAAO,CAAC,aAAa,CAAC;IAczB;;OAEG;IACG,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAcrF;;OAEG;IACG,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAQ7C;;OAEG;IACG,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKhD;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAa9D;;OAEG;IACG,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAStD;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAI7B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAG5B;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,OAAO,CAAC,EAAE,cAAc,GAAG,cAAc,CAYvE;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,CAAC,EAAE,cAAc,GAAG,kBAAkB,CAsBlF;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,aAAa,CACjC,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,cAAc,GAAG,oBAAoB,CAAC,CAchD;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IACP,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB,GACL,OAAO,CAAC,WAAW,EAAE,CAAC,CA+DxB;AAmBD;;GAEG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG;IACpD,cAAc,EAAE,WAAW,EAAE,CAAC;IAC9B,SAAS,EAAE,WAAW,EAAE,CAAC;CAC1B,CAeA;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,cAAc,GAAG,oBAAoB,GAC7C,OAAO,CAAC,IAAI,CAAC,CASf"}
|
package/dist/lib/sandbox.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Sandbox integration for isolated eval execution.
|
|
3
|
+
* Supports both Vercel Sandbox and Docker backends.
|
|
3
4
|
*/
|
|
4
5
|
import { Sandbox as VercelSandbox } from '@vercel/sandbox';
|
|
5
6
|
import { readFileSync } from 'fs';
|
|
6
7
|
import { join } from 'path';
|
|
8
|
+
import { DockerSandboxManager } from './docker-sandbox.js';
|
|
7
9
|
/**
|
|
8
10
|
* Default timeout for sandbox operations (10 minutes).
|
|
9
11
|
*/
|
|
@@ -139,6 +141,80 @@ export class SandboxManager {
|
|
|
139
141
|
await this.sandbox.stop();
|
|
140
142
|
}
|
|
141
143
|
}
|
|
144
|
+
/**
|
|
145
|
+
* Resolve which sandbox backend to use based on options.
|
|
146
|
+
*
|
|
147
|
+
* Priority:
|
|
148
|
+
* 1. Explicit backend in options (if not 'auto')
|
|
149
|
+
* 2. Auto-detect: Vercel if token present, else Docker
|
|
150
|
+
*/
|
|
151
|
+
export function resolveBackend(options) {
|
|
152
|
+
// Explicit backend in options
|
|
153
|
+
if (options?.backend && options.backend !== 'auto') {
|
|
154
|
+
return options.backend;
|
|
155
|
+
}
|
|
156
|
+
// Auto-detect: Vercel if token present, else Docker
|
|
157
|
+
if (process.env.VERCEL_TOKEN || process.env.VERCEL_OIDC_TOKEN) {
|
|
158
|
+
return 'vercel';
|
|
159
|
+
}
|
|
160
|
+
return 'docker';
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Get information about the sandbox backend that will be used.
|
|
164
|
+
* Useful for displaying to users.
|
|
165
|
+
*/
|
|
166
|
+
export function getSandboxBackendInfo(options) {
|
|
167
|
+
const backend = resolveBackend(options);
|
|
168
|
+
// Determine the reason
|
|
169
|
+
let reason;
|
|
170
|
+
let description;
|
|
171
|
+
const hasExplicitOption = options?.backend && options.backend !== 'auto';
|
|
172
|
+
if (hasExplicitOption) {
|
|
173
|
+
reason = 'explicit';
|
|
174
|
+
description = `${backend} (explicit)`;
|
|
175
|
+
}
|
|
176
|
+
else {
|
|
177
|
+
reason = 'auto-detected';
|
|
178
|
+
if (backend === 'vercel') {
|
|
179
|
+
description = `${backend} (auto-detected: VERCEL_TOKEN found)`;
|
|
180
|
+
}
|
|
181
|
+
else {
|
|
182
|
+
description = `${backend} (auto-detected: no VERCEL_TOKEN, using Docker)`;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return { backend, reason, description };
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Create a sandbox using the appropriate backend.
|
|
189
|
+
*
|
|
190
|
+
* By default, uses Vercel Sandbox if VERCEL_TOKEN is present,
|
|
191
|
+
* otherwise falls back to Docker.
|
|
192
|
+
*
|
|
193
|
+
* @example
|
|
194
|
+
* ```typescript
|
|
195
|
+
* // Auto-detect backend
|
|
196
|
+
* const sandbox = await createSandbox();
|
|
197
|
+
*
|
|
198
|
+
* // Explicit Docker
|
|
199
|
+
* const sandbox = await createSandbox({ backend: 'docker' });
|
|
200
|
+
*
|
|
201
|
+
* // Explicit Vercel
|
|
202
|
+
* const sandbox = await createSandbox({ backend: 'vercel' });
|
|
203
|
+
* ```
|
|
204
|
+
*/
|
|
205
|
+
export async function createSandbox(options = {}) {
|
|
206
|
+
const backend = resolveBackend(options);
|
|
207
|
+
if (backend === 'docker') {
|
|
208
|
+
return DockerSandboxManager.create({
|
|
209
|
+
timeout: options.timeout,
|
|
210
|
+
runtime: options.runtime,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
return SandboxManager.create({
|
|
214
|
+
timeout: options.timeout,
|
|
215
|
+
runtime: options.runtime,
|
|
216
|
+
});
|
|
217
|
+
}
|
|
142
218
|
/**
|
|
143
219
|
* Collect files from a local directory for uploading to sandbox.
|
|
144
220
|
*/
|
package/dist/lib/sandbox.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sandbox.js","sourceRoot":"","sources":["../../src/lib/sandbox.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"sandbox.js","sourceRoot":"","sources":["../../src/lib/sandbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAE3D;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAmB9C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,MAAM;IACN,OAAO;IACP,cAAc;IACd,WAAW;IACX,OAAO;IACP,OAAO;IACP,MAAM;IACN,gBAAgB;IAChB,mBAAmB;CACpB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;AA+BvE;;GAEG;AACH,MAAM,OAAO,cAAc;IACjB,OAAO,CAAgB;IACvB,iBAAiB,GAAW,iBAAiB,CAAC;IAEtD,YAAY,OAAsB;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAA0B,EAAE;QAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,uBAAuB,CAAC;QAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC;QAE5C,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QACjE,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CACd,OAAe,EACf,OAAiB,EAAE,EACnB,UAA4C,EAAE;QAE9C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;YAC3C,GAAG,EAAE,OAAO;YACZ,IAAI;YACJ,GAAG,EAAE,OAAO,CAAC,GAAG;SACjB,CAAC,CAAC;QAEH,OAAO;YACL,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,OAAe,EAAE,GAA4B;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;YAC3C,GAAG,EAAE,MAAM;YACX,IAAI,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC;YACrB,GAAG;SACJ,CAAC,CAAC;QAEH,OAAO;YACL,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAY;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACpD,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAC3D,OAAO,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAA6B;QAC5C,MAAM,YAAY,GAA6C,EAAE,CAAC;QAElE,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACpD,YAAY,CAAC,IAAI,CAAC;gBAChB,IAAI;gBACJ,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;aACvC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,KAAoB;QACpC,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,OAAO,EAAE,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO;SACrF,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC5B,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,OAAwB;IACrD,8BAA8B;IAC9B,IAAI,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QACnD,OAAO,OAAO,CAAC,OAAO,CAAC;IACzB,CAAC;IAED,oDAAoD;IACpD,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QAC9D,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAwB;IAC5D,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAExC,uBAAuB;IACvB,IAAI,MAAoC,CAAC;IACzC,IAAI,WAAmB,CAAC;IAExB,MAAM,iBAAiB,GAAG,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,CAAC;IAEzE,IAAI,iBAAiB,EAAE,CAAC;QACtB,MAAM,GAAG,UAAU,CAAC;QACpB,WAAW,GAAG,GAAG,OAAO,aAAa,CAAC;IACxC,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,eAAe,CAAC;QACzB,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YACzB,WAAW,GAAG,GAAG,OAAO,sCAAsC,CAAC;QACjE,CAAC;aAAM,CAAC;YACN,WAAW,GAAG,GAAG,OAAO,iDAAiD,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AAC1C,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA0B,EAAE;IAE5B,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAExC,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,OAAO,oBAAoB,CAAC,MAAM,CAAC;YACjC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,cAAc,CAAC,MAAM,CAAC;QAC3B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,UAGI,EAAE;IAEN,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAErD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,gBAAgB,CAAC;IACpE,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAChD,MAAM,KAAK,GAAkB,EAAE,CAAC;IAEhC,SAAS,aAAa,CAAC,IAAY,EAAE,YAAoB;QACvD,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,mBAAmB;gBACnB,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC7B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACvB,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,KAAK,OAAO,IAAI,YAAY,KAAK,OAAO,EAAE,CAAC;gBACxD,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,aAAa,CAAC,IAAY;QACjC,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC7B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACvB,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC5B,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,IAAI,CAAC,UAAkB,EAAE,eAAuB,EAAE;QACzD,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QAExC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,iBAAiB,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,YAAY,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAEzC,IAAI,aAAa,CAAC,KAAK,EAAE,iBAAiB,CAAC,EAAE,CAAC;gBAC5C,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;YACpC,CAAC;iBAAM,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;gBACvC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAgB;IACzC,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;QACzC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7B,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;aAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,KAAoB;IAIjD,MAAM,cAAc,GAAkB,EAAE,CAAC;IACzC,MAAM,SAAS,GAAkB,EAAE,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC;QAErD,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAA8C;IAE9C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CACnC,mHAAmH,CACpH,CAAC;IAEF,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IACxC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,iDAAiD,UAAU,EAAE,CAAC,CAAC;IACjF,CAAC;AACH,CAAC"}
|
package/dist/lib/types.d.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
/**
|
|
5
5
|
* Supported AI agent types.
|
|
6
6
|
*/
|
|
7
|
-
export type AgentType = 'vercel-ai-gateway/claude-code' | 'claude-code' | 'vercel-ai-gateway/codex' | 'codex';
|
|
7
|
+
export type AgentType = 'vercel-ai-gateway/claude-code' | 'claude-code' | 'vercel-ai-gateway/codex' | 'codex' | 'vercel-ai-gateway/opencode';
|
|
8
8
|
/**
|
|
9
9
|
* Model identifier - any string accepted.
|
|
10
10
|
* Each agent validates its own models at runtime.
|
|
@@ -39,6 +39,10 @@ export interface Sandbox {
|
|
|
39
39
|
* Receives a sandbox instance for pre-configuration.
|
|
40
40
|
*/
|
|
41
41
|
export type SetupFunction = (sandbox: Sandbox) => Promise<void>;
|
|
42
|
+
/**
|
|
43
|
+
* Sandbox backend type.
|
|
44
|
+
*/
|
|
45
|
+
export type SandboxBackend = 'vercel' | 'docker';
|
|
42
46
|
/**
|
|
43
47
|
* Experiment configuration.
|
|
44
48
|
* Defines what to test and how.
|
|
@@ -60,6 +64,8 @@ export interface ExperimentConfig {
|
|
|
60
64
|
timeout?: number;
|
|
61
65
|
/** Setup function that runs before agent starts. @default undefined */
|
|
62
66
|
setup?: SetupFunction;
|
|
67
|
+
/** Sandbox backend to use. @default 'auto' (Vercel if token present, else Docker) */
|
|
68
|
+
sandbox?: SandboxBackend | 'auto';
|
|
63
69
|
}
|
|
64
70
|
/**
|
|
65
71
|
* Resolved experiment config with all defaults applied.
|
|
@@ -73,6 +79,7 @@ export interface ResolvedExperimentConfig {
|
|
|
73
79
|
scripts: string[];
|
|
74
80
|
timeout: number;
|
|
75
81
|
setup?: SetupFunction;
|
|
82
|
+
sandbox: SandboxBackend | 'auto';
|
|
76
83
|
}
|
|
77
84
|
/**
|
|
78
85
|
* Required files for a valid eval fixture.
|
package/dist/lib/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,GACP,4BAA4B,CAAC;AAEjC;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;AAEnD;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,mCAAmC;IACnC,UAAU,CACR,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,GACzC,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,wCAAwC;IACxC,mBAAmB,IAAI,MAAM,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,CAAC;IAEjB,+HAA+H;IAC/H,KAAK,CAAC,EAAE,SAAS,CAAC;IAElB,+FAA+F;IAC/F,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IAEvC,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uEAAuE;IACvE,KAAK,CAAC,EAAE,aAAa,CAAC;IAEtB,qFAAqF;IACrF,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;CAClC;AAED;;;GAGG;AACH,eAAO,MAAM,mBAAmB,mDAAoD,CAAC;AAErF;;;;GAIG;AACH,eAAO,MAAM,cAAc,uEAAwE,CAAC;AAEpG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,wDAAwD;IACxD,WAAW,CAAC,EAAE;QACZ,kCAAkC;QAClC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,8DAA8D;QAC9D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0DAA0D;IAC1D,MAAM,EAAE,aAAa,CAAC;IACtB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,aAAa,CAAC,EAAE;QACd,0BAA0B;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,qDAAqD;QACrD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,IAAI,EAAE,WAAW,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,oCAAoC;IACpC,MAAM,EAAE,wBAAwB,CAAC;IACjC,4BAA4B;IAC5B,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB"}
|
package/dist/lib/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAqGH;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,cAAc,CAAU,CAAC;AAErF;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,CAAU,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vercel/agent-eval",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.6",
|
|
4
4
|
"description": "Framework for testing AI coding agents in isolated sandboxes",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -19,6 +19,10 @@
|
|
|
19
19
|
"build": "tsc",
|
|
20
20
|
"test": "vitest run",
|
|
21
21
|
"test:watch": "vitest",
|
|
22
|
+
"test:docker": "SANDBOX_BACKEND=docker vitest run src/lib/docker-sandbox.test.ts",
|
|
23
|
+
"test:integration": "INTEGRATION_TEST=1 vitest run src/integration.test.ts",
|
|
24
|
+
"test:integration:docker": "SANDBOX_BACKEND=docker INTEGRATION_TEST=1 vitest run src/integration.test.ts",
|
|
25
|
+
"test:integration:vercel": "SANDBOX_BACKEND=vercel INTEGRATION_TEST=1 vitest run src/integration.test.ts",
|
|
22
26
|
"lint": "eslint src/",
|
|
23
27
|
"prepublishOnly": "npm run build"
|
|
24
28
|
},
|
|
@@ -28,12 +32,16 @@
|
|
|
28
32
|
"ai": "^5.0.11",
|
|
29
33
|
"chalk": "^5.3.0",
|
|
30
34
|
"commander": "^12.1.0",
|
|
35
|
+
"dockerode": "^4.0.9",
|
|
31
36
|
"dotenv": "^16.4.5",
|
|
32
37
|
"glob": "^11.0.0",
|
|
38
|
+
"tar-stream": "^3.1.7",
|
|
33
39
|
"zod": "^3.23.8"
|
|
34
40
|
},
|
|
35
41
|
"devDependencies": {
|
|
42
|
+
"@types/dockerode": "^4.0.1",
|
|
36
43
|
"@types/node": "^22.0.0",
|
|
44
|
+
"@types/tar-stream": "^3.1.4",
|
|
37
45
|
"eslint": "^9.0.0",
|
|
38
46
|
"tsx": "^4.21.0",
|
|
39
47
|
"typescript": "^5.6.0",
|