@vercel/agent-eval 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/dist/lib/agents/claude-code/agent.d.ts +30 -0
  2. package/dist/lib/agents/claude-code/agent.d.ts.map +1 -0
  3. package/dist/lib/agents/claude-code/agent.js +80 -0
  4. package/dist/lib/agents/claude-code/agent.js.map +1 -0
  5. package/dist/lib/agents/claude-code/run.mjs +217 -0
  6. package/dist/lib/agents/codex/agent.d.ts +61 -0
  7. package/dist/lib/agents/codex/agent.d.ts.map +1 -0
  8. package/dist/lib/agents/codex/agent.js +221 -0
  9. package/dist/lib/agents/codex/agent.js.map +1 -0
  10. package/dist/lib/agents/codex/run.mjs +365 -0
  11. package/dist/lib/agents/cursor/agent.d.ts +32 -0
  12. package/dist/lib/agents/cursor/agent.d.ts.map +1 -0
  13. package/dist/lib/agents/cursor/agent.js +86 -0
  14. package/dist/lib/agents/cursor/agent.js.map +1 -0
  15. package/dist/lib/agents/cursor/run.mjs +183 -0
  16. package/dist/lib/agents/eval-helper.mjs +222 -0
  17. package/dist/lib/agents/gemini/agent.d.ts +27 -0
  18. package/dist/lib/agents/gemini/agent.d.ts.map +1 -0
  19. package/dist/lib/agents/gemini/agent.js +69 -0
  20. package/dist/lib/agents/gemini/agent.js.map +1 -0
  21. package/dist/lib/agents/gemini/run.mjs +180 -0
  22. package/dist/lib/agents/index.js +5 -5
  23. package/dist/lib/agents/index.js.map +1 -1
  24. package/dist/lib/agents/opencode/agent.d.ts +68 -0
  25. package/dist/lib/agents/opencode/agent.d.ts.map +1 -0
  26. package/dist/lib/agents/opencode/agent.js +165 -0
  27. package/dist/lib/agents/opencode/agent.js.map +1 -0
  28. package/dist/lib/agents/opencode/run.mjs +308 -0
  29. package/dist/lib/agents/plugin/contract.d.ts +157 -0
  30. package/dist/lib/agents/plugin/contract.d.ts.map +1 -0
  31. package/dist/lib/agents/plugin/contract.js +22 -0
  32. package/dist/lib/agents/plugin/contract.js.map +1 -0
  33. package/dist/lib/agents/plugin/orchestrator.d.ts +22 -0
  34. package/dist/lib/agents/plugin/orchestrator.d.ts.map +1 -0
  35. package/dist/lib/agents/plugin/orchestrator.js +286 -0
  36. package/dist/lib/agents/plugin/orchestrator.js.map +1 -0
  37. package/dist/lib/agents/shared.d.ts +7 -1
  38. package/dist/lib/agents/shared.d.ts.map +1 -1
  39. package/dist/lib/agents/shared.js +30 -3
  40. package/dist/lib/agents/shared.js.map +1 -1
  41. package/package.json +2 -2
  42. package/dist/lib/agents/claude-code.d.ts +0 -23
  43. package/dist/lib/agents/claude-code.d.ts.map +0 -1
  44. package/dist/lib/agents/claude-code.js +0 -300
  45. package/dist/lib/agents/claude-code.js.map +0 -1
  46. package/dist/lib/agents/codex.d.ts +0 -23
  47. package/dist/lib/agents/codex.d.ts.map +0 -1
  48. package/dist/lib/agents/codex.js +0 -364
  49. package/dist/lib/agents/codex.js.map +0 -1
  50. package/dist/lib/agents/cursor.d.ts +0 -10
  51. package/dist/lib/agents/cursor.d.ts.map +0 -1
  52. package/dist/lib/agents/cursor.js +0 -207
  53. package/dist/lib/agents/cursor.js.map +0 -1
  54. package/dist/lib/agents/gemini.d.ts +0 -10
  55. package/dist/lib/agents/gemini.d.ts.map +0 -1
  56. package/dist/lib/agents/gemini.js +0 -208
  57. package/dist/lib/agents/gemini.js.map +0 -1
  58. package/dist/lib/agents/opencode.d.ts +0 -46
  59. package/dist/lib/agents/opencode.d.ts.map +0 -1
  60. package/dist/lib/agents/opencode.js +0 -365
  61. package/dist/lib/agents/opencode.js.map +0 -1
@@ -0,0 +1,365 @@
1
+ /**
2
+ * OpenAI Codex in-sandbox runner.
3
+ *
4
+ * This file is shipped INTO the sandbox by the orchestrator and executed there as
5
+ * `node __agent_eval__/run.mjs '<AgentRunInput JSON>'`. It is intentionally
6
+ * ZERO-DEPENDENCY (only `node:*` builtins) because the sandbox only has the
7
+ * fixture's own deps + the installed `codex` CLI — it cannot import anything from
8
+ * the @vercel/agent-eval package.
9
+ *
10
+ * Dual mode:
11
+ * - runnable: invoked directly → reads argv, runs the agent, writes the result
12
+ * file + prints a status line, exits 0.
13
+ * - importable: `import { runAgent } from './run.mjs'` → returns a RunnerResult
14
+ * (no file write, no exit). This is what a future in-sandbox judge reuses.
15
+ *
16
+ * The pure helpers below are exported (not just `runAgent`) so they can be
17
+ * unit-tested directly — the same code the sandbox runs (see codex.test.ts).
18
+ *
19
+ * SECRETS: the apiKey is NOT in the AgentRunInput JSON. It arrives via process.env
20
+ * (AI_GATEWAY_API_KEY for the gateway, else OPENAI_API_KEY for direct OpenAI) and
21
+ * is piped into `codex login --with-api-key` over stdin — never on the argv.
22
+ */
23
+
24
+ import { spawnSync } from 'node:child_process';
25
+ import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
26
+ import { homedir } from 'node:os';
27
+ import { dirname, join } from 'node:path';
28
+ import { fileURLToPath } from 'node:url';
29
+
30
+ /**
31
+ * Extract transcript from Codex JSON output.
32
+ * When run with --json, Codex outputs JSONL to stdout with the full transcript.
33
+ * Preserved verbatim from the old adapter.
34
+ *
35
+ * @param {string} output
36
+ * @returns {string|undefined}
37
+ */
38
+ export function extractTranscriptFromOutput(output) {
39
+ if (!output || !output.trim()) {
40
+ return undefined;
41
+ }
42
+
43
+ // The --json output is already the transcript in JSONL format.
44
+ // Filter to only include lines that look like JSON objects.
45
+ const lines = output.split('\n').filter((line) => {
46
+ const trimmed = line.trim();
47
+ return trimmed.startsWith('{') && trimmed.endsWith('}');
48
+ });
49
+
50
+ if (lines.length === 0) {
51
+ return undefined;
52
+ }
53
+
54
+ return lines.join('\n');
55
+ }
56
+
57
+ /**
58
+ * Find the `thread.started` event's thread_id in Codex JSON stdout.
59
+ * Preserved verbatim from the old adapter. Non-JSON lines are ignored.
60
+ *
61
+ * @param {string} output
62
+ * @returns {string|undefined}
63
+ */
64
+ export function extractCodexThreadId(output) {
65
+ for (const line of output.split('\n')) {
66
+ if (!line.trim()) continue;
67
+ try {
68
+ const event = JSON.parse(line);
69
+ if (event.type === 'thread.started' && typeof event.thread_id === 'string') {
70
+ return event.thread_id;
71
+ }
72
+ } catch {
73
+ // Ignore non-JSON output lines.
74
+ }
75
+ }
76
+
77
+ return undefined;
78
+ }
79
+
80
+ /**
81
+ * Scan a Codex session transcript (JSONL) for the last `turn_context` model seen.
82
+ * Preserved verbatim from the old adapter. Non-JSON lines are ignored.
83
+ *
84
+ * @param {string|undefined} transcript
85
+ * @returns {string|undefined}
86
+ */
87
+ export function extractObservedModelFromCodexSession(transcript) {
88
+ if (!transcript) return undefined;
89
+
90
+ let observedModel;
91
+ for (const line of transcript.split('\n')) {
92
+ if (!line.trim()) continue;
93
+ try {
94
+ const event = JSON.parse(line);
95
+ const model = event.payload?.model ?? event.payload?.collaboration_mode?.settings?.model;
96
+ if (event.type === 'turn_context' && typeof model === 'string') {
97
+ observedModel = model;
98
+ }
99
+ } catch {
100
+ // Ignore non-JSON transcript lines.
101
+ }
102
+ }
103
+
104
+ return observedModel;
105
+ }
106
+
107
+ /**
108
+ * Recursively collect every `*.jsonl` file path under `~/.codex/sessions`.
109
+ * The old adapter used `find ~/.codex/sessions -type f -name '*.jsonl'`.
110
+ *
111
+ * @param {string} dir
112
+ * @returns {string[]} absolute paths
113
+ */
114
+ function listSessionFiles(dir) {
115
+ const out = [];
116
+ let entries;
117
+ try {
118
+ entries = readdirSync(dir, { withFileTypes: true });
119
+ } catch {
120
+ return out;
121
+ }
122
+ for (const entry of entries) {
123
+ const full = join(dir, entry.name);
124
+ if (entry.isDirectory()) {
125
+ out.push(...listSessionFiles(full));
126
+ } else if (entry.isFile() && entry.name.endsWith('.jsonl')) {
127
+ out.push(full);
128
+ }
129
+ }
130
+ return out;
131
+ }
132
+
133
+ /**
134
+ * Locate + read the Codex session transcript file.
135
+ *
136
+ * Codex writes JSONL session files under `~/.codex/sessions`. The old adapter, in
137
+ * shell, did:
138
+ * - with a threadId: `find ... -name '*<threadId>*.jsonl' | head -1`
139
+ * - without: `find ... -name '*.jsonl' | sort | tail -1`
140
+ *
141
+ * We mirror that exactly: filter by threadId substring when present (first match),
142
+ * otherwise lexicographically sort all `.jsonl` paths and take the last. Codex
143
+ * session filenames are timestamp-prefixed, so the lexicographic max is the newest.
144
+ *
145
+ * Best-effort: any failure → undefined (never throws).
146
+ *
147
+ * @param {string|undefined} threadId
148
+ * @returns {string|undefined}
149
+ */
150
+ function captureCodexSessionTranscript(threadId) {
151
+ try {
152
+ const sessionsDir = join(homedir(), '.codex', 'sessions');
153
+ const files = listSessionFiles(sessionsDir);
154
+ if (files.length === 0) return undefined;
155
+
156
+ let chosen;
157
+ if (threadId) {
158
+ // `find ... | head -1`: the first match in the traversal order.
159
+ chosen = files.find((f) => f.includes(threadId));
160
+ } else {
161
+ // `find ... | sort | tail -1`: lexicographic max.
162
+ chosen = files.slice().sort()[files.length - 1];
163
+ }
164
+ if (!chosen) return undefined;
165
+ return readFileSync(chosen, 'utf8');
166
+ } catch {
167
+ return undefined;
168
+ }
169
+ }
170
+
171
+ /**
172
+ * Build the `codex login --with-api-key` arg list. The key itself is piped over
173
+ * stdin (spawnSync { input }), never placed on the argv.
174
+ *
175
+ * @returns {string[]}
176
+ */
177
+ export function buildCodexLoginArgs() {
178
+ return ['login', '--with-api-key'];
179
+ }
180
+
181
+ /**
182
+ * Build the `codex exec` arg list, reproducing the old shell command verbatim:
183
+ * codex exec --profile default[ --model <m>] --dangerously-bypass-approvals-and-sandbox
184
+ * --json --skip-git-repo-check[ -c model_reasoning_effort="<e>"]
185
+ * [ -c model_verbosity="<v>"] '<prompt>'
186
+ *
187
+ * The model / reasoningEffort / verbosity values are HOST-computed (from
188
+ * parseModelString) and arrive via input.extra so they match the TOML profile.
189
+ *
190
+ * Note: with spawnSync (argv, not a shell), the `-c key="value"` tokens keep their
191
+ * literal quotes exactly as the old shell string had them, and the prompt is a
192
+ * plain argv element (no shell escaping needed).
193
+ *
194
+ * @param {{prompt:string, extra?:Record<string,unknown>}} input
195
+ * @returns {string[]}
196
+ */
197
+ export function buildCodexExecArgs(input) {
198
+ const extra = input.extra ?? {};
199
+ const cliModel = extra.cliModel ?? null;
200
+ const reasoningEffort = extra.reasoningEffort ?? null;
201
+ const verbosity = extra.verbosity ?? null;
202
+
203
+ const args = ['exec', '--profile', 'default'];
204
+ if (cliModel) {
205
+ args.push('--model', String(cliModel));
206
+ }
207
+ args.push('--dangerously-bypass-approvals-and-sandbox', '--json', '--skip-git-repo-check');
208
+ if (reasoningEffort) {
209
+ args.push('-c', `model_reasoning_effort="${reasoningEffort}"`);
210
+ }
211
+ if (verbosity) {
212
+ args.push('-c', `model_verbosity="${verbosity}"`);
213
+ }
214
+ args.push(input.prompt);
215
+ return args;
216
+ }
217
+
218
+ /**
219
+ * Run Codex over the workspace at `input.cwd` and return a RunnerResult.
220
+ *
221
+ * Two-step, mirroring the old adapter's `codex login --with-api-key && codex exec`:
222
+ * 1. `codex login --with-api-key` with the key piped on stdin. If login exits
223
+ * non-zero, short-circuit with ok:false WITHOUT running exec (the old `&&`).
224
+ * 2. `codex exec ...` — capture transcript + observedModel even on non-zero exit.
225
+ *
226
+ * Auth env (AI_GATEWAY_API_KEY / OPENAI_API_KEY) arrives via process.env — the
227
+ * orchestrator sets it on the `node run.mjs` invocation, and we pass process.env
228
+ * straight through to the CLI. The login key is read from that same env.
229
+ *
230
+ * @param {import('../plugin/contract.js').AgentRunInput} input
231
+ * @returns {Promise<import('../plugin/contract.js').RunnerResult>}
232
+ */
233
+ export async function runAgent(input) {
234
+ // The login key: gateway runs use AI_GATEWAY_API_KEY; direct OpenAI uses
235
+ // OPENAI_API_KEY. Reading both (gateway first) keeps the runner agnostic to the
236
+ // mode while never carrying the secret in the argv JSON.
237
+ const apiKey = process.env.AI_GATEWAY_API_KEY || process.env.OPENAI_API_KEY || '';
238
+
239
+ // Step 1: codex login --with-api-key (key piped on stdin). The built-in openai
240
+ // provider requires bearer auth; the gateway provider reads env_key but login is
241
+ // harmless/consistent there too. This is the left side of the old `&&`.
242
+ const login = spawnSync('codex', buildCodexLoginArgs(), {
243
+ cwd: input.cwd,
244
+ env: process.env,
245
+ input: apiKey,
246
+ encoding: 'utf8',
247
+ maxBuffer: 64 * 1024 * 1024,
248
+ });
249
+
250
+ const loginStdout = login.stdout || '';
251
+ const loginStderr = login.stderr || '';
252
+ const loginExit = login.status == null ? -1 : login.status;
253
+
254
+ if (login.error || loginExit !== 0) {
255
+ // Short-circuit: the old `codex login ... && codex exec ...` never ran exec
256
+ // when login failed. Surface the same kind of error (last 5 lines, else coded
257
+ // fallback) over the login output.
258
+ const loginOutput = loginStdout + loginStderr;
259
+ const errorLines = loginOutput.trim().split('\n').slice(-5).join('\n');
260
+ const fallback = login.error
261
+ ? `Failed to run codex: ${login.error.message}`
262
+ : `Codex CLI exited with code ${loginExit}`;
263
+ return {
264
+ ok: false,
265
+ output: loginOutput,
266
+ transcript: null,
267
+ observedModel: null,
268
+ error: errorLines || fallback,
269
+ agentExitCode: loginExit,
270
+ };
271
+ }
272
+
273
+ // Step 2: codex exec. Blocking is fine — the runner has nothing else to do while
274
+ // the agent works. The sandbox-level timeout bounds it.
275
+ const res = spawnSync('codex', buildCodexExecArgs(input), {
276
+ cwd: input.cwd,
277
+ env: process.env,
278
+ encoding: 'utf8',
279
+ maxBuffer: 64 * 1024 * 1024,
280
+ });
281
+
282
+ const stdout = res.stdout || '';
283
+ const stderr = res.stderr || '';
284
+ // Preserve the old concatenation order: stdout THEN stderr.
285
+ const output = stdout + stderr;
286
+ // spawnSync sets status=null + error when the binary can't be spawned at all.
287
+ const agentExitCode = res.status == null ? -1 : res.status;
288
+
289
+ // Capture transcript + observed model regardless of success (the old adapter did
290
+ // this even on the non-zero-exit path). The inline --json on stdout is the
291
+ // transcript; observedModel comes from the saved session file under ~/.codex.
292
+ const transcript = extractTranscriptFromOutput(output) ?? null;
293
+ const threadId = extractCodexThreadId(output);
294
+ const sessionTranscript = captureCodexSessionTranscript(threadId);
295
+ const observedModel = extractObservedModelFromCodexSession(sessionTranscript) ?? null;
296
+
297
+ if (res.error || agentExitCode !== 0) {
298
+ // Mirror the old error string: last 5 lines of output, else a coded fallback.
299
+ const errorLines = output.trim().split('\n').slice(-5).join('\n');
300
+ const fallback = res.error
301
+ ? `Failed to run codex: ${res.error.message}`
302
+ : `Codex CLI exited with code ${agentExitCode}`;
303
+ return {
304
+ ok: false,
305
+ output,
306
+ transcript,
307
+ observedModel,
308
+ error: errorLines || fallback,
309
+ agentExitCode,
310
+ };
311
+ }
312
+
313
+ return { ok: true, output, transcript, observedModel, error: null, agentExitCode };
314
+ }
315
+
316
+ /* ─────────────────────────── runnable (CLI) entry ─────────────────────────── */
317
+
318
+ // True when this file is executed directly (`node run.mjs ...`), false when imported.
319
+ const isMain = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
320
+
321
+ if (isMain) {
322
+ // argv[2] is the AgentRunInput JSON (never contains secrets).
323
+ const input = JSON.parse(process.argv[2]);
324
+
325
+ // Always produce a RunnerResult, even if the runner itself throws, so the host
326
+ // always has a result file to read (node exit code stays 0 except on a truly
327
+ // unrecoverable crash before we can write).
328
+ let result;
329
+ try {
330
+ result = await runAgent(input);
331
+ } catch (e) {
332
+ result = {
333
+ ok: false,
334
+ output: '',
335
+ transcript: null,
336
+ observedModel: null,
337
+ error: e && e.message ? e.message : String(e),
338
+ agentExitCode: -1,
339
+ };
340
+ }
341
+
342
+ // Source of truth: the result file the host reads back via sandbox.readFile.
343
+ try {
344
+ mkdirSync(dirname(input.resultPath), { recursive: true });
345
+ writeFileSync(input.resultPath, JSON.stringify(result));
346
+ } catch {
347
+ // If the file can't be written, the host falls back to the marker line below.
348
+ }
349
+
350
+ // Fallback channel: a compact status line (no transcript — it can be huge).
351
+ process.stdout.write(
352
+ '__AGENT_RESULT__ ' +
353
+ JSON.stringify({
354
+ ok: result.ok,
355
+ observedModel: result.observedModel,
356
+ error: result.error,
357
+ agentExitCode: result.agentExitCode,
358
+ }) +
359
+ '\n'
360
+ );
361
+
362
+ // Exit 0: "the runner ran". Agent success/failure is conveyed via result.ok, not
363
+ // the node exit code (the host distinguishes the two).
364
+ process.exit(0);
365
+ }
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Cursor CLI agent — host-side definition + the thin Agent wrapper.
3
+ *
4
+ * The definition is pure data/auth; the actual CLI invocation + transcript capture
5
+ * live in ./run.mjs (shipped into the sandbox by the orchestrator). The wrapper
6
+ * keeps the public Agent interface identical so the registry / runner are untouched.
7
+ *
8
+ * Cursor uses direct Cursor API access only (no gateway branch), so auth is a single
9
+ * env var (CURSOR_API_KEY). It writes no config files — everything is CLI flags + env.
10
+ */
11
+ import type { Agent } from '../types.js';
12
+ import type { AgentDefinition } from '../plugin/contract.js';
13
+ /**
14
+ * Build the Cursor CLI plugin definition.
15
+ *
16
+ * Preserved exactly from the old adapter:
17
+ * - install: project `npm install` (retried once) then the official install script
18
+ * `curl https://cursor.com/install -fsSL | bash` (a SHELL step, since it pipes).
19
+ * - no config files.
20
+ * - auth: CURSOR_API_KEY only (direct API, no gateway mode).
21
+ * - o11y parser name: 'cursor'.
22
+ * - default model tier: 'composer-1.5'.
23
+ */
24
+ export declare function createCursorDefinition(): AgentDefinition;
25
+ /**
26
+ * Create the Cursor CLI Agent. Thin wrapper over the generic orchestrator so the
27
+ * Agent interface (and thus registry.ts / index.ts / runner.ts) is unchanged.
28
+ *
29
+ * NOTE: keeps the old no-arg signature — index.ts calls `createCursorAgent()`.
30
+ */
31
+ export declare function createCursorAgent(): Agent;
32
+ //# sourceMappingURL=agent.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../../../src/lib/agents/cursor/agent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EAAE,KAAK,EAAmB,MAAM,aAAa,CAAC;AAG1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAG7D;;;;;;;;;;GAUG;AACH,wBAAgB,sBAAsB,IAAI,eAAe,CA8CxD;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,IAAI,KAAK,CAWzC"}
@@ -0,0 +1,86 @@
1
+ /**
2
+ * Cursor CLI agent — host-side definition + the thin Agent wrapper.
3
+ *
4
+ * The definition is pure data/auth; the actual CLI invocation + transcript capture
5
+ * live in ./run.mjs (shipped into the sandbox by the orchestrator). The wrapper
6
+ * keeps the public Agent interface identical so the registry / runner are untouched.
7
+ *
8
+ * Cursor uses direct Cursor API access only (no gateway branch), so auth is a single
9
+ * env var (CURSOR_API_KEY). It writes no config files — everything is CLI flags + env.
10
+ */
11
+ import { fileURLToPath } from 'node:url';
12
+ import { CURSOR_DIRECT } from '../shared.js';
13
+ import { runWithDefinition } from '../plugin/orchestrator.js';
14
+ /**
15
+ * Build the Cursor CLI plugin definition.
16
+ *
17
+ * Preserved exactly from the old adapter:
18
+ * - install: project `npm install` (retried once) then the official install script
19
+ * `curl https://cursor.com/install -fsSL | bash` (a SHELL step, since it pipes).
20
+ * - no config files.
21
+ * - auth: CURSOR_API_KEY only (direct API, no gateway mode).
22
+ * - o11y parser name: 'cursor'.
23
+ * - default model tier: 'composer-1.5'.
24
+ */
25
+ export function createCursorDefinition() {
26
+ return {
27
+ name: 'cursor',
28
+ displayName: 'Cursor CLI',
29
+ defaultModel: 'composer-1.5',
30
+ o11yAgentName: 'cursor',
31
+ // Resolve run.mjs next to this file (works in src during dev and in dist after
32
+ // the build copies run.mjs alongside the compiled agent.js).
33
+ runnerPath: fileURLToPath(new URL('./run.mjs', import.meta.url)),
34
+ getApiKeyEnvVar() {
35
+ return CURSOR_DIRECT.apiKeyEnvVar;
36
+ },
37
+ install(_options) {
38
+ // Project deps (retried once), then the Cursor CLI via the official installer.
39
+ // The installer pipes curl→bash, so it must be a 'shell' step (not 'command').
40
+ return [
41
+ {
42
+ kind: 'command',
43
+ cmd: 'npm',
44
+ args: ['install'],
45
+ retryOnce: true,
46
+ errorPrefix: 'npm install failed',
47
+ errorBody: 'last10',
48
+ },
49
+ {
50
+ kind: 'shell',
51
+ script: 'curl https://cursor.com/install -fsSL | bash',
52
+ errorPrefix: 'Cursor CLI install failed',
53
+ errorBody: 'stderr',
54
+ },
55
+ ];
56
+ },
57
+ // Cursor is configured purely via CLI flags + env — no config files.
58
+ configFiles() {
59
+ return [];
60
+ },
61
+ authEnv(options) {
62
+ // Direct API only: a single key env var. (The neutral-workspace env is merged
63
+ // on top by the orchestrator.)
64
+ return { [CURSOR_DIRECT.apiKeyEnvVar]: options.apiKey };
65
+ },
66
+ };
67
+ }
68
+ /**
69
+ * Create the Cursor CLI Agent. Thin wrapper over the generic orchestrator so the
70
+ * Agent interface (and thus registry.ts / index.ts / runner.ts) is unchanged.
71
+ *
72
+ * NOTE: keeps the old no-arg signature — index.ts calls `createCursorAgent()`.
73
+ */
74
+ export function createCursorAgent() {
75
+ const definition = createCursorDefinition();
76
+ return {
77
+ name: definition.name,
78
+ displayName: definition.displayName,
79
+ getApiKeyEnvVar: definition.getApiKeyEnvVar,
80
+ getDefaultModel() {
81
+ return definition.defaultModel;
82
+ },
83
+ run: (fixturePath, options) => runWithDefinition(definition, fixturePath, options),
84
+ };
85
+ }
86
+ //# sourceMappingURL=agent.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"agent.js","sourceRoot":"","sources":["../../../../src/lib/agents/cursor/agent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D;;;;;;;;;;GAUG;AACH,MAAM,UAAU,sBAAsB;IACpC,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,WAAW,EAAE,YAAY;QACzB,YAAY,EAAE,cAAc;QAC5B,aAAa,EAAE,QAAQ;QACvB,+EAA+E;QAC/E,6DAA6D;QAC7D,UAAU,EAAE,aAAa,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEhE,eAAe;YACb,OAAO,aAAa,CAAC,YAAY,CAAC;QACpC,CAAC;QAED,OAAO,CAAC,QAAyB;YAC/B,+EAA+E;YAC/E,+EAA+E;YAC/E,OAAO;gBACL;oBACE,IAAI,EAAE,SAAS;oBACf,GAAG,EAAE,KAAK;oBACV,IAAI,EAAE,CAAC,SAAS,CAAC;oBACjB,SAAS,EAAE,IAAI;oBACf,WAAW,EAAE,oBAAoB;oBACjC,SAAS,EAAE,QAAQ;iBACpB;gBACD;oBACE,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,8CAA8C;oBACtD,WAAW,EAAE,2BAA2B;oBACxC,SAAS,EAAE,QAAQ;iBACpB;aACF,CAAC;QACJ,CAAC;QAED,qEAAqE;QACrE,WAAW;YACT,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,CAAC,OAAwB;YAC9B,8EAA8E;YAC9E,+BAA+B;YAC/B,OAAO,EAAE,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QAC1D,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB;IAC/B,MAAM,UAAU,GAAG,sBAAsB,EAAE,CAAC;IAC5C,OAAO;QACL,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,eAAe,EAAE,UAAU,CAAC,eAAe;QAC3C,eAAe;YACb,OAAO,UAAU,CAAC,YAAY,CAAC;QACjC,CAAC;QACD,GAAG,EAAE,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC;KACnF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,183 @@
1
+ /**
2
+ * Cursor CLI in-sandbox runner.
3
+ *
4
+ * This file is shipped INTO the sandbox by the orchestrator and executed there as
5
+ * `node __agent_eval__/run.mjs '<AgentRunInput JSON>'`. It is intentionally
6
+ * ZERO-DEPENDENCY (only `node:*` builtins) because the sandbox only has the
7
+ * fixture's own deps + the installed `agent` (Cursor) CLI — it cannot import
8
+ * anything from the @vercel/agent-eval package.
9
+ *
10
+ * Dual mode:
11
+ * - runnable: invoked directly → reads argv, runs the agent, writes the result
12
+ * file + prints a status line, exits 0.
13
+ * - importable: `import { runAgent } from './run.mjs'` → returns a RunnerResult
14
+ * (no file write, no exit). This is what a future in-sandbox judge reuses.
15
+ *
16
+ * The pure helpers below are exported (not just `runAgent`) so they can be
17
+ * unit-tested directly — the same code the sandbox runs.
18
+ *
19
+ * Cursor's transcript is the `--output-format stream-json` JSONL it prints to
20
+ * stdout/stderr — there is no on-disk session file to read, and (like gemini) no
21
+ * observed-model extraction, so observedModel is always null.
22
+ */
23
+
24
+ import { spawnSync } from 'node:child_process';
25
+ import { mkdirSync, writeFileSync } from 'node:fs';
26
+ import { dirname } from 'node:path';
27
+ import { fileURLToPath } from 'node:url';
28
+
29
+ /**
30
+ * Build the Cursor CLI argument list.
31
+ *
32
+ * Order is regression-sensitive (preserved verbatim from the old adapter):
33
+ * - the prompt is the FIRST positional argument.
34
+ * - `--print`: non-interactive mode (required for scripts/headless).
35
+ * - `--force`: auto-approve all tool operations.
36
+ * - `--model`: only when a model override is provided.
37
+ * - `--output-format stream-json`: structured JSONL transcript (only works with
38
+ * --print). Emitted LAST, exactly as the old adapter did.
39
+ *
40
+ * @param {{prompt:string, model?:string}} input
41
+ * @returns {string[]}
42
+ */
43
+ export function buildCursorCliArgs(input) {
44
+ const cursorArgs = [input.prompt, '--print', '--force'];
45
+ if (input.model) {
46
+ cursorArgs.push('--model', input.model);
47
+ }
48
+ cursorArgs.push('--output-format', 'stream-json');
49
+ return cursorArgs;
50
+ }
51
+
52
+ /**
53
+ * Extract transcript from Cursor CLI stream-json output.
54
+ * When run with `--output-format stream-json`, Cursor outputs JSONL
55
+ * (newline-delimited JSON). Preserved verbatim from the old adapter: keep only the
56
+ * lines that look like a complete JSON object (start with `{` and end with `}`).
57
+ *
58
+ * @param {string|undefined|null} output
59
+ * @returns {string|undefined}
60
+ */
61
+ export function extractTranscriptFromOutput(output) {
62
+ if (!output || !output.trim()) {
63
+ return undefined;
64
+ }
65
+
66
+ // The --output-format stream-json output contains JSON events, one per line.
67
+ // Filter to only include lines that look like JSON objects.
68
+ const lines = output.split('\n').filter((line) => {
69
+ const trimmed = line.trim();
70
+ return trimmed.startsWith('{') && trimmed.endsWith('}');
71
+ });
72
+
73
+ if (lines.length === 0) {
74
+ return undefined;
75
+ }
76
+
77
+ return lines.join('\n');
78
+ }
79
+
80
+ /**
81
+ * Run Cursor over the workspace at `input.cwd` and return a RunnerResult.
82
+ *
83
+ * Auth (CURSOR_API_KEY) arrives via process.env — the orchestrator sets it on the
84
+ * `node run.mjs` invocation, and we pass process.env straight through to the CLI.
85
+ * The runner never handles secrets itself.
86
+ *
87
+ * @param {import('../plugin/contract.js').AgentRunInput} input
88
+ * @returns {Promise<import('../plugin/contract.js').RunnerResult>}
89
+ */
90
+ export async function runAgent(input) {
91
+ const args = buildCursorCliArgs(input);
92
+
93
+ // spawnSync (not a shell string): the prompt is a plain argv element, so there is
94
+ // no shell quoting/escaping to get wrong. Blocking is fine — the runner has
95
+ // nothing else to do while the agent works. The sandbox-level timeout bounds it.
96
+ // The Cursor binary is named `agent` (preserved from the old adapter).
97
+ const res = spawnSync('agent', args, {
98
+ cwd: input.cwd,
99
+ env: process.env,
100
+ encoding: 'utf8',
101
+ maxBuffer: 64 * 1024 * 1024,
102
+ });
103
+
104
+ const stdout = res.stdout || '';
105
+ const stderr = res.stderr || '';
106
+ // Preserve the old concatenation order: stdout THEN stderr.
107
+ const output = stdout + stderr;
108
+ // spawnSync sets status=null + error when the binary can't be spawned at all.
109
+ const agentExitCode = res.status == null ? -1 : res.status;
110
+
111
+ // Transcript from the stream-json output; Cursor has no observed-model extraction.
112
+ const transcript = extractTranscriptFromOutput(output) ?? null;
113
+ const observedModel = null;
114
+
115
+ if (res.error || agentExitCode !== 0) {
116
+ // Mirror the old error string: last 5 lines of output, else a coded fallback.
117
+ const errorLines = output.trim().split('\n').slice(-5).join('\n');
118
+ const fallback = res.error
119
+ ? `Failed to run agent: ${res.error.message}`
120
+ : `Cursor CLI exited with code ${agentExitCode}`;
121
+ return {
122
+ ok: false,
123
+ output,
124
+ transcript,
125
+ observedModel,
126
+ error: errorLines || fallback,
127
+ agentExitCode,
128
+ };
129
+ }
130
+
131
+ return { ok: true, output, transcript, observedModel, error: null, agentExitCode };
132
+ }
133
+
134
+ /* ─────────────────────────── runnable (CLI) entry ─────────────────────────── */
135
+
136
+ // True when this file is executed directly (`node run.mjs ...`), false when imported.
137
+ const isMain = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
138
+
139
+ if (isMain) {
140
+ // argv[2] is the AgentRunInput JSON (never contains secrets).
141
+ const input = JSON.parse(process.argv[2]);
142
+
143
+ // Always produce a RunnerResult, even if the runner itself throws, so the host
144
+ // always has a result file to read (node exit code stays 0 except on a truly
145
+ // unrecoverable crash before we can write).
146
+ let result;
147
+ try {
148
+ result = await runAgent(input);
149
+ } catch (e) {
150
+ result = {
151
+ ok: false,
152
+ output: '',
153
+ transcript: null,
154
+ observedModel: null,
155
+ error: e && e.message ? e.message : String(e),
156
+ agentExitCode: -1,
157
+ };
158
+ }
159
+
160
+ // Source of truth: the result file the host reads back via sandbox.readFile.
161
+ try {
162
+ mkdirSync(dirname(input.resultPath), { recursive: true });
163
+ writeFileSync(input.resultPath, JSON.stringify(result));
164
+ } catch {
165
+ // If the file can't be written, the host falls back to the marker line below.
166
+ }
167
+
168
+ // Fallback channel: a compact status line (no transcript — it can be huge).
169
+ process.stdout.write(
170
+ '__AGENT_RESULT__ ' +
171
+ JSON.stringify({
172
+ ok: result.ok,
173
+ observedModel: result.observedModel,
174
+ error: result.error,
175
+ agentExitCode: result.agentExitCode,
176
+ }) +
177
+ '\n'
178
+ );
179
+
180
+ // Exit 0: "the runner ran". Agent success/failure is conveyed via result.ok, not
181
+ // the node exit code (the host distinguishes the two).
182
+ process.exit(0);
183
+ }