@vercel/agent-eval 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -3
- package/dist/cli.js +4 -0
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/agents/claude-code.d.ts.map +1 -1
- package/dist/lib/agents/claude-code.js +4 -8
- package/dist/lib/agents/claude-code.js.map +1 -1
- package/dist/lib/agents/codex.d.ts.map +1 -1
- package/dist/lib/agents/codex.js +4 -8
- package/dist/lib/agents/codex.js.map +1 -1
- package/dist/lib/agents/index.d.ts.map +1 -1
- package/dist/lib/agents/index.js +2 -0
- package/dist/lib/agents/index.js.map +1 -1
- package/dist/lib/agents/opencode.d.ts +11 -0
- package/dist/lib/agents/opencode.d.ts.map +1 -0
- package/dist/lib/agents/opencode.js +220 -0
- package/dist/lib/agents/opencode.js.map +1 -0
- package/dist/lib/agents/shared.d.ts +7 -3
- package/dist/lib/agents/shared.d.ts.map +1 -1
- package/dist/lib/agents/shared.js.map +1 -1
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/config.js +1 -0
- package/dist/lib/config.js.map +1 -1
- package/dist/lib/docker-sandbox.d.ts +91 -0
- package/dist/lib/docker-sandbox.d.ts.map +1 -0
- package/dist/lib/docker-sandbox.js +321 -0
- package/dist/lib/docker-sandbox.js.map +1 -0
- package/dist/lib/sandbox.d.ts +54 -2
- package/dist/lib/sandbox.d.ts.map +1 -1
- package/dist/lib/sandbox.js +85 -1
- package/dist/lib/sandbox.js.map +1 -1
- package/dist/lib/types.d.ts +1 -1
- package/dist/lib/types.d.ts.map +1 -1
- package/dist/lib/types.js.map +1 -1
- package/package.json +9 -1
package/dist/lib/sandbox.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Sandbox integration for isolated eval execution.
|
|
3
|
+
* Supports both Vercel Sandbox and Docker backends.
|
|
3
4
|
*/
|
|
4
5
|
import { Sandbox as VercelSandbox } from '@vercel/sandbox';
|
|
5
6
|
import { readFileSync } from 'fs';
|
|
6
7
|
import { join } from 'path';
|
|
8
|
+
import { DockerSandboxManager } from './docker-sandbox.js';
|
|
7
9
|
/**
|
|
8
10
|
* Default timeout for sandbox operations (10 minutes).
|
|
9
11
|
*/
|
|
@@ -139,6 +141,88 @@ export class SandboxManager {
|
|
|
139
141
|
await this.sandbox.stop();
|
|
140
142
|
}
|
|
141
143
|
}
|
|
144
|
+
/**
|
|
145
|
+
* Resolve which sandbox backend to use based on options and environment.
|
|
146
|
+
*
|
|
147
|
+
* Priority:
|
|
148
|
+
* 1. Explicit backend in options (if not 'auto')
|
|
149
|
+
* 2. SANDBOX_BACKEND environment variable
|
|
150
|
+
* 3. Auto-detect: Vercel if token present, else Docker
|
|
151
|
+
*/
|
|
152
|
+
export function resolveBackend(options) {
|
|
153
|
+
// Explicit backend in options
|
|
154
|
+
if (options?.backend && options.backend !== 'auto') {
|
|
155
|
+
return options.backend;
|
|
156
|
+
}
|
|
157
|
+
// Environment variable override
|
|
158
|
+
const envBackend = process.env.SANDBOX_BACKEND;
|
|
159
|
+
if (envBackend === 'vercel' || envBackend === 'docker') {
|
|
160
|
+
return envBackend;
|
|
161
|
+
}
|
|
162
|
+
// Auto-detect: Vercel if token present, else Docker
|
|
163
|
+
if (process.env.VERCEL_TOKEN || process.env.VERCEL_OIDC_TOKEN) {
|
|
164
|
+
return 'vercel';
|
|
165
|
+
}
|
|
166
|
+
return 'docker';
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Get information about the sandbox backend that will be used.
|
|
170
|
+
* Useful for displaying to users.
|
|
171
|
+
*/
|
|
172
|
+
export function getSandboxBackendInfo(options) {
|
|
173
|
+
const backend = resolveBackend(options);
|
|
174
|
+
// Determine the reason
|
|
175
|
+
let reason;
|
|
176
|
+
let description;
|
|
177
|
+
const envBackend = process.env.SANDBOX_BACKEND;
|
|
178
|
+
const hasExplicitOption = options?.backend && options.backend !== 'auto';
|
|
179
|
+
const hasEnvVar = envBackend === 'vercel' || envBackend === 'docker';
|
|
180
|
+
if (hasExplicitOption || hasEnvVar) {
|
|
181
|
+
reason = 'explicit';
|
|
182
|
+
description = `${backend} (explicit)`;
|
|
183
|
+
}
|
|
184
|
+
else {
|
|
185
|
+
reason = 'auto-detected';
|
|
186
|
+
if (backend === 'vercel') {
|
|
187
|
+
description = `${backend} (auto-detected: VERCEL_TOKEN found)`;
|
|
188
|
+
}
|
|
189
|
+
else {
|
|
190
|
+
description = `${backend} (auto-detected: no VERCEL_TOKEN, using Docker)`;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
return { backend, reason, description };
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Create a sandbox using the appropriate backend.
|
|
197
|
+
*
|
|
198
|
+
* By default, uses Vercel Sandbox if VERCEL_TOKEN is present,
|
|
199
|
+
* otherwise falls back to Docker.
|
|
200
|
+
*
|
|
201
|
+
* @example
|
|
202
|
+
* ```typescript
|
|
203
|
+
* // Auto-detect backend
|
|
204
|
+
* const sandbox = await createSandbox();
|
|
205
|
+
*
|
|
206
|
+
* // Explicit Docker
|
|
207
|
+
* const sandbox = await createSandbox({ backend: 'docker' });
|
|
208
|
+
*
|
|
209
|
+
* // Explicit Vercel
|
|
210
|
+
* const sandbox = await createSandbox({ backend: 'vercel' });
|
|
211
|
+
* ```
|
|
212
|
+
*/
|
|
213
|
+
export async function createSandbox(options = {}) {
|
|
214
|
+
const backend = resolveBackend(options);
|
|
215
|
+
if (backend === 'docker') {
|
|
216
|
+
return DockerSandboxManager.create({
|
|
217
|
+
timeout: options.timeout,
|
|
218
|
+
runtime: options.runtime,
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
return SandboxManager.create({
|
|
222
|
+
timeout: options.timeout,
|
|
223
|
+
runtime: options.runtime,
|
|
224
|
+
});
|
|
225
|
+
}
|
|
142
226
|
/**
|
|
143
227
|
* Collect files from a local directory for uploading to sandbox.
|
|
144
228
|
*/
|
package/dist/lib/sandbox.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sandbox.js","sourceRoot":"","sources":["../../src/lib/sandbox.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"sandbox.js","sourceRoot":"","sources":["../../src/lib/sandbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAE3D;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAmB9C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,MAAM;IACN,OAAO;IACP,cAAc;IACd,WAAW;IACX,OAAO;IACP,OAAO;IACP,MAAM;IACN,gBAAgB;IAChB,mBAAmB;CACpB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;AA+BvE;;GAEG;AACH,MAAM,OAAO,cAAc;IACjB,OAAO,CAAgB;IACvB,iBAAiB,GAAW,iBAAiB,CAAC;IAEtD,YAAY,OAAsB;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAA0B,EAAE;QAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,uBAAuB,CAAC;QAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC;QAE5C,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QACjE,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CACd,OAAe,EACf,OAAiB,EAAE,EACnB,UAA4C,EAAE;QAE9C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;YAC3C,GAAG,EAAE,OAAO;YACZ,IAAI;YACJ,GAAG,EAAE,OAAO,CAAC,GAAG;SACjB,CAAC,CAAC;QAEH,OAAO;YACL,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,OAAe,EAAE,GAA4B;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;YAC3C,GAAG,EAAE,MAAM;YACX,IAAI,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC;YACrB,GAAG;SACJ,CAAC,CAAC;QAEH,OAAO;YACL,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,EAAE;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAY;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACpD,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAC3D,OAAO,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAA6B;QAC5C,MAAM,YAAY,GAA6C,EAAE,CAAC;QAElE,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACpD,YAAY,CAAC,IAAI,CAAC;gBAChB,IAAI;gBACJ,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;aACvC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,KAAoB;QACpC,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,OAAO,EAAE,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO;SACrF,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC5B,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAC,OAAwB;IACrD,8BAA8B;IAC9B,IAAI,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QACnD,OAAO,OAAO,CAAC,OAAO,CAAC;IACzB,CAAC;IAED,gCAAgC;IAChC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IAC/C,IAAI,UAAU,KAAK,QAAQ,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;QACvD,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,oDAAoD;IACpD,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QAC9D,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAwB;IAC5D,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAExC,uBAAuB;IACvB,IAAI,MAAoC,CAAC;IACzC,IAAI,WAAmB,CAAC;IAExB,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IAC/C,MAAM,iBAAiB,GAAG,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,CAAC;IACzE,MAAM,SAAS,GAAG,UAAU,KAAK,QAAQ,IAAI,UAAU,KAAK,QAAQ,CAAC;IAErE,IAAI,iBAAiB,IAAI,SAAS,EAAE,CAAC;QACnC,MAAM,GAAG,UAAU,CAAC;QACpB,WAAW,GAAG,GAAG,OAAO,aAAa,CAAC;IACxC,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,eAAe,CAAC;QACzB,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YACzB,WAAW,GAAG,GAAG,OAAO,sCAAsC,CAAC;QACjE,CAAC;aAAM,CAAC;YACN,WAAW,GAAG,GAAG,OAAO,iDAAiD,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AAC1C,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA0B,EAAE;IAE5B,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAExC,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,OAAO,oBAAoB,CAAC,MAAM,CAAC;YACjC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,cAAc,CAAC,MAAM,CAAC;QAC3B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,UAGI,EAAE;IAEN,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAErD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,gBAAgB,CAAC;IACpE,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAChD,MAAM,KAAK,GAAkB,EAAE,CAAC;IAEhC,SAAS,aAAa,CAAC,IAAY,EAAE,YAAoB;QACvD,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,mBAAmB;gBACnB,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC7B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACvB,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,KAAK,OAAO,IAAI,YAAY,KAAK,OAAO,EAAE,CAAC;gBACxD,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,aAAa,CAAC,IAAY;QACjC,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC7B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACvB,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC5B,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,IAAI,CAAC,UAAkB,EAAE,eAAuB,EAAE;QACzD,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QAExC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,iBAAiB,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,YAAY,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAEzC,IAAI,aAAa,CAAC,KAAK,EAAE,iBAAiB,CAAC,EAAE,CAAC;gBAC5C,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;YACpC,CAAC;iBAAM,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;gBACvC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAgB;IACzC,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;QACzC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7B,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;aAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,KAAoB;IAIjD,MAAM,cAAc,GAAkB,EAAE,CAAC;IACzC,MAAM,SAAS,GAAkB,EAAE,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC;QAErD,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAA8C;IAE9C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CACnC,mHAAmH,CACpH,CAAC;IAEF,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IACxC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,iDAAiD,UAAU,EAAE,CAAC,CAAC;IACjF,CAAC;AACH,CAAC"}
|
package/dist/lib/types.d.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
/**
|
|
5
5
|
* Supported AI agent types.
|
|
6
6
|
*/
|
|
7
|
-
export type AgentType = 'vercel-ai-gateway/claude-code' | 'claude-code' | 'vercel-ai-gateway/codex' | 'codex';
|
|
7
|
+
export type AgentType = 'vercel-ai-gateway/claude-code' | 'claude-code' | 'vercel-ai-gateway/codex' | 'codex' | 'vercel-ai-gateway/opencode';
|
|
8
8
|
/**
|
|
9
9
|
* Model identifier - any string accepted.
|
|
10
10
|
* Each agent validates its own models at runtime.
|
package/dist/lib/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,GACP,4BAA4B,CAAC;AAEjC;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;AAEnD;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,mCAAmC;IACnC,UAAU,CACR,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,GACzC,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,wCAAwC;IACxC,mBAAmB,IAAI,MAAM,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEhE;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,CAAC;IAEjB,+HAA+H;IAC/H,KAAK,CAAC,EAAE,SAAS,CAAC;IAElB,+FAA+F;IAC/F,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IAEvC,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uEAAuE;IACvE,KAAK,CAAC,EAAE,aAAa,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;CACvB;AAED;;;GAGG;AACH,eAAO,MAAM,mBAAmB,mDAAoD,CAAC;AAErF;;;;GAIG;AACH,eAAO,MAAM,cAAc,uEAAwE,CAAC;AAEpG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,wDAAwD;IACxD,WAAW,CAAC,EAAE;QACZ,kCAAkC;QAClC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,8DAA8D;QAC9D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0DAA0D;IAC1D,MAAM,EAAE,aAAa,CAAC;IACtB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,aAAa,CAAC,EAAE;QACd,0BAA0B;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,qDAAqD;QACrD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,IAAI,EAAE,WAAW,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,oCAAoC;IACpC,MAAM,EAAE,wBAAwB,CAAC;IACjC,4BAA4B;IAC5B,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB"}
|
package/dist/lib/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA4FH;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,cAAc,CAAU,CAAC;AAErF;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,CAAU,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vercel/agent-eval",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.5",
|
|
4
4
|
"description": "Framework for testing AI coding agents in isolated sandboxes",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -19,6 +19,10 @@
|
|
|
19
19
|
"build": "tsc",
|
|
20
20
|
"test": "vitest run",
|
|
21
21
|
"test:watch": "vitest",
|
|
22
|
+
"test:docker": "SANDBOX_BACKEND=docker vitest run src/lib/docker-sandbox.test.ts",
|
|
23
|
+
"test:integration": "INTEGRATION_TEST=1 vitest run src/integration.test.ts",
|
|
24
|
+
"test:integration:docker": "SANDBOX_BACKEND=docker INTEGRATION_TEST=1 vitest run src/integration.test.ts",
|
|
25
|
+
"test:integration:vercel": "SANDBOX_BACKEND=vercel INTEGRATION_TEST=1 vitest run src/integration.test.ts",
|
|
22
26
|
"lint": "eslint src/",
|
|
23
27
|
"prepublishOnly": "npm run build"
|
|
24
28
|
},
|
|
@@ -28,12 +32,16 @@
|
|
|
28
32
|
"ai": "^5.0.11",
|
|
29
33
|
"chalk": "^5.3.0",
|
|
30
34
|
"commander": "^12.1.0",
|
|
35
|
+
"dockerode": "^4.0.9",
|
|
31
36
|
"dotenv": "^16.4.5",
|
|
32
37
|
"glob": "^11.0.0",
|
|
38
|
+
"tar-stream": "^3.1.7",
|
|
33
39
|
"zod": "^3.23.8"
|
|
34
40
|
},
|
|
35
41
|
"devDependencies": {
|
|
42
|
+
"@types/dockerode": "^4.0.1",
|
|
36
43
|
"@types/node": "^22.0.0",
|
|
44
|
+
"@types/tar-stream": "^3.1.4",
|
|
37
45
|
"eslint": "^9.0.0",
|
|
38
46
|
"tsx": "^4.21.0",
|
|
39
47
|
"typescript": "^5.6.0",
|