@vercel/agent-eval 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/agents/claude-code/agent.d.ts +30 -0
- package/dist/lib/agents/claude-code/agent.d.ts.map +1 -0
- package/dist/lib/agents/claude-code/agent.js +80 -0
- package/dist/lib/agents/claude-code/agent.js.map +1 -0
- package/dist/lib/agents/claude-code/run.mjs +217 -0
- package/dist/lib/agents/codex/agent.d.ts +61 -0
- package/dist/lib/agents/codex/agent.d.ts.map +1 -0
- package/dist/lib/agents/codex/agent.js +221 -0
- package/dist/lib/agents/codex/agent.js.map +1 -0
- package/dist/lib/agents/codex/run.mjs +365 -0
- package/dist/lib/agents/cursor/agent.d.ts +32 -0
- package/dist/lib/agents/cursor/agent.d.ts.map +1 -0
- package/dist/lib/agents/cursor/agent.js +86 -0
- package/dist/lib/agents/cursor/agent.js.map +1 -0
- package/dist/lib/agents/cursor/run.mjs +183 -0
- package/dist/lib/agents/eval-helper.mjs +222 -0
- package/dist/lib/agents/gemini/agent.d.ts +27 -0
- package/dist/lib/agents/gemini/agent.d.ts.map +1 -0
- package/dist/lib/agents/gemini/agent.js +69 -0
- package/dist/lib/agents/gemini/agent.js.map +1 -0
- package/dist/lib/agents/gemini/run.mjs +180 -0
- package/dist/lib/agents/index.js +5 -5
- package/dist/lib/agents/index.js.map +1 -1
- package/dist/lib/agents/opencode/agent.d.ts +68 -0
- package/dist/lib/agents/opencode/agent.d.ts.map +1 -0
- package/dist/lib/agents/opencode/agent.js +165 -0
- package/dist/lib/agents/opencode/agent.js.map +1 -0
- package/dist/lib/agents/opencode/run.mjs +308 -0
- package/dist/lib/agents/plugin/contract.d.ts +157 -0
- package/dist/lib/agents/plugin/contract.d.ts.map +1 -0
- package/dist/lib/agents/plugin/contract.js +22 -0
- package/dist/lib/agents/plugin/contract.js.map +1 -0
- package/dist/lib/agents/plugin/orchestrator.d.ts +22 -0
- package/dist/lib/agents/plugin/orchestrator.d.ts.map +1 -0
- package/dist/lib/agents/plugin/orchestrator.js +286 -0
- package/dist/lib/agents/plugin/orchestrator.js.map +1 -0
- package/dist/lib/agents/shared.d.ts +7 -1
- package/dist/lib/agents/shared.d.ts.map +1 -1
- package/dist/lib/agents/shared.js +30 -3
- package/dist/lib/agents/shared.js.map +1 -1
- package/package.json +2 -2
- package/dist/lib/agents/claude-code.d.ts +0 -23
- package/dist/lib/agents/claude-code.d.ts.map +0 -1
- package/dist/lib/agents/claude-code.js +0 -300
- package/dist/lib/agents/claude-code.js.map +0 -1
- package/dist/lib/agents/codex.d.ts +0 -23
- package/dist/lib/agents/codex.d.ts.map +0 -1
- package/dist/lib/agents/codex.js +0 -364
- package/dist/lib/agents/codex.js.map +0 -1
- package/dist/lib/agents/cursor.d.ts +0 -10
- package/dist/lib/agents/cursor.d.ts.map +0 -1
- package/dist/lib/agents/cursor.js +0 -207
- package/dist/lib/agents/cursor.js.map +0 -1
- package/dist/lib/agents/gemini.d.ts +0 -10
- package/dist/lib/agents/gemini.d.ts.map +0 -1
- package/dist/lib/agents/gemini.js +0 -208
- package/dist/lib/agents/gemini.js.map +0 -1
- package/dist/lib/agents/opencode.d.ts +0 -46
- package/dist/lib/agents/opencode.d.ts.map +0 -1
- package/dist/lib/agents/opencode.js +0 -365
- package/dist/lib/agents/opencode.js.map +0 -1
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenAI Codex in-sandbox runner.
|
|
3
|
+
*
|
|
4
|
+
* This file is shipped INTO the sandbox by the orchestrator and executed there as
|
|
5
|
+
* `node __agent_eval__/run.mjs '<AgentRunInput JSON>'`. It is intentionally
|
|
6
|
+
* ZERO-DEPENDENCY (only `node:*` builtins) because the sandbox only has the
|
|
7
|
+
* fixture's own deps + the installed `codex` CLI — it cannot import anything from
|
|
8
|
+
* the @vercel/agent-eval package.
|
|
9
|
+
*
|
|
10
|
+
* Dual mode:
|
|
11
|
+
* - runnable: invoked directly → reads argv, runs the agent, writes the result
|
|
12
|
+
* file + prints a status line, exits 0.
|
|
13
|
+
* - importable: `import { runAgent } from './run.mjs'` → returns a RunnerResult
|
|
14
|
+
* (no file write, no exit). This is what a future in-sandbox judge reuses.
|
|
15
|
+
*
|
|
16
|
+
* The pure helpers below are exported (not just `runAgent`) so they can be
|
|
17
|
+
* unit-tested directly — the same code the sandbox runs (see codex.test.ts).
|
|
18
|
+
*
|
|
19
|
+
* SECRETS: the apiKey is NOT in the AgentRunInput JSON. It arrives via process.env
|
|
20
|
+
* (AI_GATEWAY_API_KEY for the gateway, else OPENAI_API_KEY for direct OpenAI) and
|
|
21
|
+
* is piped into `codex login --with-api-key` over stdin — never on the argv.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { spawnSync } from 'node:child_process';
|
|
25
|
+
import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
26
|
+
import { homedir } from 'node:os';
|
|
27
|
+
import { dirname, join } from 'node:path';
|
|
28
|
+
import { fileURLToPath } from 'node:url';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Extract transcript from Codex JSON output.
|
|
32
|
+
* When run with --json, Codex outputs JSONL to stdout with the full transcript.
|
|
33
|
+
* Preserved verbatim from the old adapter.
|
|
34
|
+
*
|
|
35
|
+
* @param {string} output
|
|
36
|
+
* @returns {string|undefined}
|
|
37
|
+
*/
|
|
38
|
+
export function extractTranscriptFromOutput(output) {
|
|
39
|
+
if (!output || !output.trim()) {
|
|
40
|
+
return undefined;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// The --json output is already the transcript in JSONL format.
|
|
44
|
+
// Filter to only include lines that look like JSON objects.
|
|
45
|
+
const lines = output.split('\n').filter((line) => {
|
|
46
|
+
const trimmed = line.trim();
|
|
47
|
+
return trimmed.startsWith('{') && trimmed.endsWith('}');
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
if (lines.length === 0) {
|
|
51
|
+
return undefined;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return lines.join('\n');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Find the `thread.started` event's thread_id in Codex JSON stdout.
|
|
59
|
+
* Preserved verbatim from the old adapter. Non-JSON lines are ignored.
|
|
60
|
+
*
|
|
61
|
+
* @param {string} output
|
|
62
|
+
* @returns {string|undefined}
|
|
63
|
+
*/
|
|
64
|
+
export function extractCodexThreadId(output) {
|
|
65
|
+
for (const line of output.split('\n')) {
|
|
66
|
+
if (!line.trim()) continue;
|
|
67
|
+
try {
|
|
68
|
+
const event = JSON.parse(line);
|
|
69
|
+
if (event.type === 'thread.started' && typeof event.thread_id === 'string') {
|
|
70
|
+
return event.thread_id;
|
|
71
|
+
}
|
|
72
|
+
} catch {
|
|
73
|
+
// Ignore non-JSON output lines.
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Scan a Codex session transcript (JSONL) for the last `turn_context` model seen.
|
|
82
|
+
* Preserved verbatim from the old adapter. Non-JSON lines are ignored.
|
|
83
|
+
*
|
|
84
|
+
* @param {string|undefined} transcript
|
|
85
|
+
* @returns {string|undefined}
|
|
86
|
+
*/
|
|
87
|
+
export function extractObservedModelFromCodexSession(transcript) {
|
|
88
|
+
if (!transcript) return undefined;
|
|
89
|
+
|
|
90
|
+
let observedModel;
|
|
91
|
+
for (const line of transcript.split('\n')) {
|
|
92
|
+
if (!line.trim()) continue;
|
|
93
|
+
try {
|
|
94
|
+
const event = JSON.parse(line);
|
|
95
|
+
const model = event.payload?.model ?? event.payload?.collaboration_mode?.settings?.model;
|
|
96
|
+
if (event.type === 'turn_context' && typeof model === 'string') {
|
|
97
|
+
observedModel = model;
|
|
98
|
+
}
|
|
99
|
+
} catch {
|
|
100
|
+
// Ignore non-JSON transcript lines.
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
return observedModel;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Recursively collect every `*.jsonl` file path under `~/.codex/sessions`.
|
|
109
|
+
* The old adapter used `find ~/.codex/sessions -type f -name '*.jsonl'`.
|
|
110
|
+
*
|
|
111
|
+
* @param {string} dir
|
|
112
|
+
* @returns {string[]} absolute paths
|
|
113
|
+
*/
|
|
114
|
+
function listSessionFiles(dir) {
|
|
115
|
+
const out = [];
|
|
116
|
+
let entries;
|
|
117
|
+
try {
|
|
118
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
119
|
+
} catch {
|
|
120
|
+
return out;
|
|
121
|
+
}
|
|
122
|
+
for (const entry of entries) {
|
|
123
|
+
const full = join(dir, entry.name);
|
|
124
|
+
if (entry.isDirectory()) {
|
|
125
|
+
out.push(...listSessionFiles(full));
|
|
126
|
+
} else if (entry.isFile() && entry.name.endsWith('.jsonl')) {
|
|
127
|
+
out.push(full);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return out;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Locate + read the Codex session transcript file.
|
|
135
|
+
*
|
|
136
|
+
* Codex writes JSONL session files under `~/.codex/sessions`. The old adapter, in
|
|
137
|
+
* shell, did:
|
|
138
|
+
* - with a threadId: `find ... -name '*<threadId>*.jsonl' | head -1`
|
|
139
|
+
* - without: `find ... -name '*.jsonl' | sort | tail -1`
|
|
140
|
+
*
|
|
141
|
+
* We mirror that exactly: filter by threadId substring when present (first match),
|
|
142
|
+
* otherwise lexicographically sort all `.jsonl` paths and take the last. Codex
|
|
143
|
+
* session filenames are timestamp-prefixed, so the lexicographic max is the newest.
|
|
144
|
+
*
|
|
145
|
+
* Best-effort: any failure → undefined (never throws).
|
|
146
|
+
*
|
|
147
|
+
* @param {string|undefined} threadId
|
|
148
|
+
* @returns {string|undefined}
|
|
149
|
+
*/
|
|
150
|
+
function captureCodexSessionTranscript(threadId) {
|
|
151
|
+
try {
|
|
152
|
+
const sessionsDir = join(homedir(), '.codex', 'sessions');
|
|
153
|
+
const files = listSessionFiles(sessionsDir);
|
|
154
|
+
if (files.length === 0) return undefined;
|
|
155
|
+
|
|
156
|
+
let chosen;
|
|
157
|
+
if (threadId) {
|
|
158
|
+
// `find ... | head -1`: the first match in the traversal order.
|
|
159
|
+
chosen = files.find((f) => f.includes(threadId));
|
|
160
|
+
} else {
|
|
161
|
+
// `find ... | sort | tail -1`: lexicographic max.
|
|
162
|
+
chosen = files.slice().sort()[files.length - 1];
|
|
163
|
+
}
|
|
164
|
+
if (!chosen) return undefined;
|
|
165
|
+
return readFileSync(chosen, 'utf8');
|
|
166
|
+
} catch {
|
|
167
|
+
return undefined;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Build the `codex login --with-api-key` arg list. The key itself is piped over
|
|
173
|
+
* stdin (spawnSync { input }), never placed on the argv.
|
|
174
|
+
*
|
|
175
|
+
* @returns {string[]}
|
|
176
|
+
*/
|
|
177
|
+
export function buildCodexLoginArgs() {
|
|
178
|
+
return ['login', '--with-api-key'];
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Build the `codex exec` arg list, reproducing the old shell command verbatim:
|
|
183
|
+
* codex exec --profile default[ --model <m>] --dangerously-bypass-approvals-and-sandbox
|
|
184
|
+
* --json --skip-git-repo-check[ -c model_reasoning_effort="<e>"]
|
|
185
|
+
* [ -c model_verbosity="<v>"] '<prompt>'
|
|
186
|
+
*
|
|
187
|
+
* The model / reasoningEffort / verbosity values are HOST-computed (from
|
|
188
|
+
* parseModelString) and arrive via input.extra so they match the TOML profile.
|
|
189
|
+
*
|
|
190
|
+
* Note: with spawnSync (argv, not a shell), the `-c key="value"` tokens keep their
|
|
191
|
+
* literal quotes exactly as the old shell string had them, and the prompt is a
|
|
192
|
+
* plain argv element (no shell escaping needed).
|
|
193
|
+
*
|
|
194
|
+
* @param {{prompt:string, extra?:Record<string,unknown>}} input
|
|
195
|
+
* @returns {string[]}
|
|
196
|
+
*/
|
|
197
|
+
export function buildCodexExecArgs(input) {
|
|
198
|
+
const extra = input.extra ?? {};
|
|
199
|
+
const cliModel = extra.cliModel ?? null;
|
|
200
|
+
const reasoningEffort = extra.reasoningEffort ?? null;
|
|
201
|
+
const verbosity = extra.verbosity ?? null;
|
|
202
|
+
|
|
203
|
+
const args = ['exec', '--profile', 'default'];
|
|
204
|
+
if (cliModel) {
|
|
205
|
+
args.push('--model', String(cliModel));
|
|
206
|
+
}
|
|
207
|
+
args.push('--dangerously-bypass-approvals-and-sandbox', '--json', '--skip-git-repo-check');
|
|
208
|
+
if (reasoningEffort) {
|
|
209
|
+
args.push('-c', `model_reasoning_effort="${reasoningEffort}"`);
|
|
210
|
+
}
|
|
211
|
+
if (verbosity) {
|
|
212
|
+
args.push('-c', `model_verbosity="${verbosity}"`);
|
|
213
|
+
}
|
|
214
|
+
args.push(input.prompt);
|
|
215
|
+
return args;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Run Codex over the workspace at `input.cwd` and return a RunnerResult.
|
|
220
|
+
*
|
|
221
|
+
* Two-step, mirroring the old adapter's `codex login --with-api-key && codex exec`:
|
|
222
|
+
* 1. `codex login --with-api-key` with the key piped on stdin. If login exits
|
|
223
|
+
* non-zero, short-circuit with ok:false WITHOUT running exec (the old `&&`).
|
|
224
|
+
* 2. `codex exec ...` — capture transcript + observedModel even on non-zero exit.
|
|
225
|
+
*
|
|
226
|
+
* Auth env (AI_GATEWAY_API_KEY / OPENAI_API_KEY) arrives via process.env — the
|
|
227
|
+
* orchestrator sets it on the `node run.mjs` invocation, and we pass process.env
|
|
228
|
+
* straight through to the CLI. The login key is read from that same env.
|
|
229
|
+
*
|
|
230
|
+
* @param {import('../plugin/contract.js').AgentRunInput} input
|
|
231
|
+
* @returns {Promise<import('../plugin/contract.js').RunnerResult>}
|
|
232
|
+
*/
|
|
233
|
+
export async function runAgent(input) {
|
|
234
|
+
// The login key: gateway runs use AI_GATEWAY_API_KEY; direct OpenAI uses
|
|
235
|
+
// OPENAI_API_KEY. Reading both (gateway first) keeps the runner agnostic to the
|
|
236
|
+
// mode while never carrying the secret in the argv JSON.
|
|
237
|
+
const apiKey = process.env.AI_GATEWAY_API_KEY || process.env.OPENAI_API_KEY || '';
|
|
238
|
+
|
|
239
|
+
// Step 1: codex login --with-api-key (key piped on stdin). The built-in openai
|
|
240
|
+
// provider requires bearer auth; the gateway provider reads env_key but login is
|
|
241
|
+
// harmless/consistent there too. This is the left side of the old `&&`.
|
|
242
|
+
const login = spawnSync('codex', buildCodexLoginArgs(), {
|
|
243
|
+
cwd: input.cwd,
|
|
244
|
+
env: process.env,
|
|
245
|
+
input: apiKey,
|
|
246
|
+
encoding: 'utf8',
|
|
247
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
const loginStdout = login.stdout || '';
|
|
251
|
+
const loginStderr = login.stderr || '';
|
|
252
|
+
const loginExit = login.status == null ? -1 : login.status;
|
|
253
|
+
|
|
254
|
+
if (login.error || loginExit !== 0) {
|
|
255
|
+
// Short-circuit: the old `codex login ... && codex exec ...` never ran exec
|
|
256
|
+
// when login failed. Surface the same kind of error (last 5 lines, else coded
|
|
257
|
+
// fallback) over the login output.
|
|
258
|
+
const loginOutput = loginStdout + loginStderr;
|
|
259
|
+
const errorLines = loginOutput.trim().split('\n').slice(-5).join('\n');
|
|
260
|
+
const fallback = login.error
|
|
261
|
+
? `Failed to run codex: ${login.error.message}`
|
|
262
|
+
: `Codex CLI exited with code ${loginExit}`;
|
|
263
|
+
return {
|
|
264
|
+
ok: false,
|
|
265
|
+
output: loginOutput,
|
|
266
|
+
transcript: null,
|
|
267
|
+
observedModel: null,
|
|
268
|
+
error: errorLines || fallback,
|
|
269
|
+
agentExitCode: loginExit,
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Step 2: codex exec. Blocking is fine — the runner has nothing else to do while
|
|
274
|
+
// the agent works. The sandbox-level timeout bounds it.
|
|
275
|
+
const res = spawnSync('codex', buildCodexExecArgs(input), {
|
|
276
|
+
cwd: input.cwd,
|
|
277
|
+
env: process.env,
|
|
278
|
+
encoding: 'utf8',
|
|
279
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
const stdout = res.stdout || '';
|
|
283
|
+
const stderr = res.stderr || '';
|
|
284
|
+
// Preserve the old concatenation order: stdout THEN stderr.
|
|
285
|
+
const output = stdout + stderr;
|
|
286
|
+
// spawnSync sets status=null + error when the binary can't be spawned at all.
|
|
287
|
+
const agentExitCode = res.status == null ? -1 : res.status;
|
|
288
|
+
|
|
289
|
+
// Capture transcript + observed model regardless of success (the old adapter did
|
|
290
|
+
// this even on the non-zero-exit path). The inline --json on stdout is the
|
|
291
|
+
// transcript; observedModel comes from the saved session file under ~/.codex.
|
|
292
|
+
const transcript = extractTranscriptFromOutput(output) ?? null;
|
|
293
|
+
const threadId = extractCodexThreadId(output);
|
|
294
|
+
const sessionTranscript = captureCodexSessionTranscript(threadId);
|
|
295
|
+
const observedModel = extractObservedModelFromCodexSession(sessionTranscript) ?? null;
|
|
296
|
+
|
|
297
|
+
if (res.error || agentExitCode !== 0) {
|
|
298
|
+
// Mirror the old error string: last 5 lines of output, else a coded fallback.
|
|
299
|
+
const errorLines = output.trim().split('\n').slice(-5).join('\n');
|
|
300
|
+
const fallback = res.error
|
|
301
|
+
? `Failed to run codex: ${res.error.message}`
|
|
302
|
+
: `Codex CLI exited with code ${agentExitCode}`;
|
|
303
|
+
return {
|
|
304
|
+
ok: false,
|
|
305
|
+
output,
|
|
306
|
+
transcript,
|
|
307
|
+
observedModel,
|
|
308
|
+
error: errorLines || fallback,
|
|
309
|
+
agentExitCode,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
return { ok: true, output, transcript, observedModel, error: null, agentExitCode };
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/* ─────────────────────────── runnable (CLI) entry ─────────────────────────── */
|
|
317
|
+
|
|
318
|
+
// True when this file is executed directly (`node run.mjs ...`), false when imported.
|
|
319
|
+
const isMain = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
|
|
320
|
+
|
|
321
|
+
if (isMain) {
|
|
322
|
+
// argv[2] is the AgentRunInput JSON (never contains secrets).
|
|
323
|
+
const input = JSON.parse(process.argv[2]);
|
|
324
|
+
|
|
325
|
+
// Always produce a RunnerResult, even if the runner itself throws, so the host
|
|
326
|
+
// always has a result file to read (node exit code stays 0 except on a truly
|
|
327
|
+
// unrecoverable crash before we can write).
|
|
328
|
+
let result;
|
|
329
|
+
try {
|
|
330
|
+
result = await runAgent(input);
|
|
331
|
+
} catch (e) {
|
|
332
|
+
result = {
|
|
333
|
+
ok: false,
|
|
334
|
+
output: '',
|
|
335
|
+
transcript: null,
|
|
336
|
+
observedModel: null,
|
|
337
|
+
error: e && e.message ? e.message : String(e),
|
|
338
|
+
agentExitCode: -1,
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// Source of truth: the result file the host reads back via sandbox.readFile.
|
|
343
|
+
try {
|
|
344
|
+
mkdirSync(dirname(input.resultPath), { recursive: true });
|
|
345
|
+
writeFileSync(input.resultPath, JSON.stringify(result));
|
|
346
|
+
} catch {
|
|
347
|
+
// If the file can't be written, the host falls back to the marker line below.
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
// Fallback channel: a compact status line (no transcript — it can be huge).
|
|
351
|
+
process.stdout.write(
|
|
352
|
+
'__AGENT_RESULT__ ' +
|
|
353
|
+
JSON.stringify({
|
|
354
|
+
ok: result.ok,
|
|
355
|
+
observedModel: result.observedModel,
|
|
356
|
+
error: result.error,
|
|
357
|
+
agentExitCode: result.agentExitCode,
|
|
358
|
+
}) +
|
|
359
|
+
'\n'
|
|
360
|
+
);
|
|
361
|
+
|
|
362
|
+
// Exit 0: "the runner ran". Agent success/failure is conveyed via result.ok, not
|
|
363
|
+
// the node exit code (the host distinguishes the two).
|
|
364
|
+
process.exit(0);
|
|
365
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cursor CLI agent — host-side definition + the thin Agent wrapper.
|
|
3
|
+
*
|
|
4
|
+
* The definition is pure data/auth; the actual CLI invocation + transcript capture
|
|
5
|
+
* live in ./run.mjs (shipped into the sandbox by the orchestrator). The wrapper
|
|
6
|
+
* keeps the public Agent interface identical so the registry / runner are untouched.
|
|
7
|
+
*
|
|
8
|
+
* Cursor uses direct Cursor API access only (no gateway branch), so auth is a single
|
|
9
|
+
* env var (CURSOR_API_KEY). It writes no config files — everything is CLI flags + env.
|
|
10
|
+
*/
|
|
11
|
+
import type { Agent } from '../types.js';
|
|
12
|
+
import type { AgentDefinition } from '../plugin/contract.js';
|
|
13
|
+
/**
|
|
14
|
+
* Build the Cursor CLI plugin definition.
|
|
15
|
+
*
|
|
16
|
+
* Preserved exactly from the old adapter:
|
|
17
|
+
* - install: project `npm install` (retried once) then the official install script
|
|
18
|
+
* `curl https://cursor.com/install -fsSL | bash` (a SHELL step, since it pipes).
|
|
19
|
+
* - no config files.
|
|
20
|
+
* - auth: CURSOR_API_KEY only (direct API, no gateway mode).
|
|
21
|
+
* - o11y parser name: 'cursor'.
|
|
22
|
+
* - default model tier: 'composer-1.5'.
|
|
23
|
+
*/
|
|
24
|
+
export declare function createCursorDefinition(): AgentDefinition;
|
|
25
|
+
/**
|
|
26
|
+
* Create the Cursor CLI Agent. Thin wrapper over the generic orchestrator so the
|
|
27
|
+
* Agent interface (and thus registry.ts / index.ts / runner.ts) is unchanged.
|
|
28
|
+
*
|
|
29
|
+
* NOTE: keeps the old no-arg signature — index.ts calls `createCursorAgent()`.
|
|
30
|
+
*/
|
|
31
|
+
export declare function createCursorAgent(): Agent;
|
|
32
|
+
//# sourceMappingURL=agent.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../../../src/lib/agents/cursor/agent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EAAE,KAAK,EAAmB,MAAM,aAAa,CAAC;AAG1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAG7D;;;;;;;;;;GAUG;AACH,wBAAgB,sBAAsB,IAAI,eAAe,CA8CxD;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,IAAI,KAAK,CAWzC"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cursor CLI agent — host-side definition + the thin Agent wrapper.
|
|
3
|
+
*
|
|
4
|
+
* The definition is pure data/auth; the actual CLI invocation + transcript capture
|
|
5
|
+
* live in ./run.mjs (shipped into the sandbox by the orchestrator). The wrapper
|
|
6
|
+
* keeps the public Agent interface identical so the registry / runner are untouched.
|
|
7
|
+
*
|
|
8
|
+
* Cursor uses direct Cursor API access only (no gateway branch), so auth is a single
|
|
9
|
+
* env var (CURSOR_API_KEY). It writes no config files — everything is CLI flags + env.
|
|
10
|
+
*/
|
|
11
|
+
import { fileURLToPath } from 'node:url';
|
|
12
|
+
import { CURSOR_DIRECT } from '../shared.js';
|
|
13
|
+
import { runWithDefinition } from '../plugin/orchestrator.js';
|
|
14
|
+
/**
|
|
15
|
+
* Build the Cursor CLI plugin definition.
|
|
16
|
+
*
|
|
17
|
+
* Preserved exactly from the old adapter:
|
|
18
|
+
* - install: project `npm install` (retried once) then the official install script
|
|
19
|
+
* `curl https://cursor.com/install -fsSL | bash` (a SHELL step, since it pipes).
|
|
20
|
+
* - no config files.
|
|
21
|
+
* - auth: CURSOR_API_KEY only (direct API, no gateway mode).
|
|
22
|
+
* - o11y parser name: 'cursor'.
|
|
23
|
+
* - default model tier: 'composer-1.5'.
|
|
24
|
+
*/
|
|
25
|
+
export function createCursorDefinition() {
|
|
26
|
+
return {
|
|
27
|
+
name: 'cursor',
|
|
28
|
+
displayName: 'Cursor CLI',
|
|
29
|
+
defaultModel: 'composer-1.5',
|
|
30
|
+
o11yAgentName: 'cursor',
|
|
31
|
+
// Resolve run.mjs next to this file (works in src during dev and in dist after
|
|
32
|
+
// the build copies run.mjs alongside the compiled agent.js).
|
|
33
|
+
runnerPath: fileURLToPath(new URL('./run.mjs', import.meta.url)),
|
|
34
|
+
getApiKeyEnvVar() {
|
|
35
|
+
return CURSOR_DIRECT.apiKeyEnvVar;
|
|
36
|
+
},
|
|
37
|
+
install(_options) {
|
|
38
|
+
// Project deps (retried once), then the Cursor CLI via the official installer.
|
|
39
|
+
// The installer pipes curl→bash, so it must be a 'shell' step (not 'command').
|
|
40
|
+
return [
|
|
41
|
+
{
|
|
42
|
+
kind: 'command',
|
|
43
|
+
cmd: 'npm',
|
|
44
|
+
args: ['install'],
|
|
45
|
+
retryOnce: true,
|
|
46
|
+
errorPrefix: 'npm install failed',
|
|
47
|
+
errorBody: 'last10',
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
kind: 'shell',
|
|
51
|
+
script: 'curl https://cursor.com/install -fsSL | bash',
|
|
52
|
+
errorPrefix: 'Cursor CLI install failed',
|
|
53
|
+
errorBody: 'stderr',
|
|
54
|
+
},
|
|
55
|
+
];
|
|
56
|
+
},
|
|
57
|
+
// Cursor is configured purely via CLI flags + env — no config files.
|
|
58
|
+
configFiles() {
|
|
59
|
+
return [];
|
|
60
|
+
},
|
|
61
|
+
authEnv(options) {
|
|
62
|
+
// Direct API only: a single key env var. (The neutral-workspace env is merged
|
|
63
|
+
// on top by the orchestrator.)
|
|
64
|
+
return { [CURSOR_DIRECT.apiKeyEnvVar]: options.apiKey };
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Create the Cursor CLI Agent. Thin wrapper over the generic orchestrator so the
|
|
70
|
+
* Agent interface (and thus registry.ts / index.ts / runner.ts) is unchanged.
|
|
71
|
+
*
|
|
72
|
+
* NOTE: keeps the old no-arg signature — index.ts calls `createCursorAgent()`.
|
|
73
|
+
*/
|
|
74
|
+
export function createCursorAgent() {
|
|
75
|
+
const definition = createCursorDefinition();
|
|
76
|
+
return {
|
|
77
|
+
name: definition.name,
|
|
78
|
+
displayName: definition.displayName,
|
|
79
|
+
getApiKeyEnvVar: definition.getApiKeyEnvVar,
|
|
80
|
+
getDefaultModel() {
|
|
81
|
+
return definition.defaultModel;
|
|
82
|
+
},
|
|
83
|
+
run: (fixturePath, options) => runWithDefinition(definition, fixturePath, options),
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
//# sourceMappingURL=agent.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent.js","sourceRoot":"","sources":["../../../../src/lib/agents/cursor/agent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D;;;;;;;;;;GAUG;AACH,MAAM,UAAU,sBAAsB;IACpC,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,WAAW,EAAE,YAAY;QACzB,YAAY,EAAE,cAAc;QAC5B,aAAa,EAAE,QAAQ;QACvB,+EAA+E;QAC/E,6DAA6D;QAC7D,UAAU,EAAE,aAAa,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEhE,eAAe;YACb,OAAO,aAAa,CAAC,YAAY,CAAC;QACpC,CAAC;QAED,OAAO,CAAC,QAAyB;YAC/B,+EAA+E;YAC/E,+EAA+E;YAC/E,OAAO;gBACL;oBACE,IAAI,EAAE,SAAS;oBACf,GAAG,EAAE,KAAK;oBACV,IAAI,EAAE,CAAC,SAAS,CAAC;oBACjB,SAAS,EAAE,IAAI;oBACf,WAAW,EAAE,oBAAoB;oBACjC,SAAS,EAAE,QAAQ;iBACpB;gBACD;oBACE,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,8CAA8C;oBACtD,WAAW,EAAE,2BAA2B;oBACxC,SAAS,EAAE,QAAQ;iBACpB;aACF,CAAC;QACJ,CAAC;QAED,qEAAqE;QACrE,WAAW;YACT,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,CAAC,OAAwB;YAC9B,8EAA8E;YAC9E,+BAA+B;YAC/B,OAAO,EAAE,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QAC1D,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB;IAC/B,MAAM,UAAU,GAAG,sBAAsB,EAAE,CAAC;IAC5C,OAAO;QACL,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,eAAe,EAAE,UAAU,CAAC,eAAe;QAC3C,eAAe;YACb,OAAO,UAAU,CAAC,YAAY,CAAC;QACjC,CAAC;QACD,GAAG,EAAE,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC;KACnF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cursor CLI in-sandbox runner.
|
|
3
|
+
*
|
|
4
|
+
* This file is shipped INTO the sandbox by the orchestrator and executed there as
|
|
5
|
+
* `node __agent_eval__/run.mjs '<AgentRunInput JSON>'`. It is intentionally
|
|
6
|
+
* ZERO-DEPENDENCY (only `node:*` builtins) because the sandbox only has the
|
|
7
|
+
* fixture's own deps + the installed `agent` (Cursor) CLI — it cannot import
|
|
8
|
+
* anything from the @vercel/agent-eval package.
|
|
9
|
+
*
|
|
10
|
+
* Dual mode:
|
|
11
|
+
* - runnable: invoked directly → reads argv, runs the agent, writes the result
|
|
12
|
+
* file + prints a status line, exits 0.
|
|
13
|
+
* - importable: `import { runAgent } from './run.mjs'` → returns a RunnerResult
|
|
14
|
+
* (no file write, no exit). This is what a future in-sandbox judge reuses.
|
|
15
|
+
*
|
|
16
|
+
* The pure helpers below are exported (not just `runAgent`) so they can be
|
|
17
|
+
* unit-tested directly — the same code the sandbox runs.
|
|
18
|
+
*
|
|
19
|
+
* Cursor's transcript is the `--output-format stream-json` JSONL it prints to
|
|
20
|
+
* stdout/stderr — there is no on-disk session file to read, and (like gemini) no
|
|
21
|
+
* observed-model extraction, so observedModel is always null.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { spawnSync } from 'node:child_process';
|
|
25
|
+
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
26
|
+
import { dirname } from 'node:path';
|
|
27
|
+
import { fileURLToPath } from 'node:url';
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Build the Cursor CLI argument list.
|
|
31
|
+
*
|
|
32
|
+
* Order is regression-sensitive (preserved verbatim from the old adapter):
|
|
33
|
+
* - the prompt is the FIRST positional argument.
|
|
34
|
+
* - `--print`: non-interactive mode (required for scripts/headless).
|
|
35
|
+
* - `--force`: auto-approve all tool operations.
|
|
36
|
+
* - `--model`: only when a model override is provided.
|
|
37
|
+
* - `--output-format stream-json`: structured JSONL transcript (only works with
|
|
38
|
+
* --print). Emitted LAST, exactly as the old adapter did.
|
|
39
|
+
*
|
|
40
|
+
* @param {{prompt:string, model?:string}} input
|
|
41
|
+
* @returns {string[]}
|
|
42
|
+
*/
|
|
43
|
+
export function buildCursorCliArgs(input) {
|
|
44
|
+
const cursorArgs = [input.prompt, '--print', '--force'];
|
|
45
|
+
if (input.model) {
|
|
46
|
+
cursorArgs.push('--model', input.model);
|
|
47
|
+
}
|
|
48
|
+
cursorArgs.push('--output-format', 'stream-json');
|
|
49
|
+
return cursorArgs;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Extract transcript from Cursor CLI stream-json output.
|
|
54
|
+
* When run with `--output-format stream-json`, Cursor outputs JSONL
|
|
55
|
+
* (newline-delimited JSON). Preserved verbatim from the old adapter: keep only the
|
|
56
|
+
* lines that look like a complete JSON object (start with `{` and end with `}`).
|
|
57
|
+
*
|
|
58
|
+
* @param {string|undefined|null} output
|
|
59
|
+
* @returns {string|undefined}
|
|
60
|
+
*/
|
|
61
|
+
export function extractTranscriptFromOutput(output) {
|
|
62
|
+
if (!output || !output.trim()) {
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// The --output-format stream-json output contains JSON events, one per line.
|
|
67
|
+
// Filter to only include lines that look like JSON objects.
|
|
68
|
+
const lines = output.split('\n').filter((line) => {
|
|
69
|
+
const trimmed = line.trim();
|
|
70
|
+
return trimmed.startsWith('{') && trimmed.endsWith('}');
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
if (lines.length === 0) {
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return lines.join('\n');
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Run Cursor over the workspace at `input.cwd` and return a RunnerResult.
|
|
82
|
+
*
|
|
83
|
+
* Auth (CURSOR_API_KEY) arrives via process.env — the orchestrator sets it on the
|
|
84
|
+
* `node run.mjs` invocation, and we pass process.env straight through to the CLI.
|
|
85
|
+
* The runner never handles secrets itself.
|
|
86
|
+
*
|
|
87
|
+
* @param {import('../plugin/contract.js').AgentRunInput} input
|
|
88
|
+
* @returns {Promise<import('../plugin/contract.js').RunnerResult>}
|
|
89
|
+
*/
|
|
90
|
+
export async function runAgent(input) {
|
|
91
|
+
const args = buildCursorCliArgs(input);
|
|
92
|
+
|
|
93
|
+
// spawnSync (not a shell string): the prompt is a plain argv element, so there is
|
|
94
|
+
// no shell quoting/escaping to get wrong. Blocking is fine — the runner has
|
|
95
|
+
// nothing else to do while the agent works. The sandbox-level timeout bounds it.
|
|
96
|
+
// The Cursor binary is named `agent` (preserved from the old adapter).
|
|
97
|
+
const res = spawnSync('agent', args, {
|
|
98
|
+
cwd: input.cwd,
|
|
99
|
+
env: process.env,
|
|
100
|
+
encoding: 'utf8',
|
|
101
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
const stdout = res.stdout || '';
|
|
105
|
+
const stderr = res.stderr || '';
|
|
106
|
+
// Preserve the old concatenation order: stdout THEN stderr.
|
|
107
|
+
const output = stdout + stderr;
|
|
108
|
+
// spawnSync sets status=null + error when the binary can't be spawned at all.
|
|
109
|
+
const agentExitCode = res.status == null ? -1 : res.status;
|
|
110
|
+
|
|
111
|
+
// Transcript from the stream-json output; Cursor has no observed-model extraction.
|
|
112
|
+
const transcript = extractTranscriptFromOutput(output) ?? null;
|
|
113
|
+
const observedModel = null;
|
|
114
|
+
|
|
115
|
+
if (res.error || agentExitCode !== 0) {
|
|
116
|
+
// Mirror the old error string: last 5 lines of output, else a coded fallback.
|
|
117
|
+
const errorLines = output.trim().split('\n').slice(-5).join('\n');
|
|
118
|
+
const fallback = res.error
|
|
119
|
+
? `Failed to run agent: ${res.error.message}`
|
|
120
|
+
: `Cursor CLI exited with code ${agentExitCode}`;
|
|
121
|
+
return {
|
|
122
|
+
ok: false,
|
|
123
|
+
output,
|
|
124
|
+
transcript,
|
|
125
|
+
observedModel,
|
|
126
|
+
error: errorLines || fallback,
|
|
127
|
+
agentExitCode,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
return { ok: true, output, transcript, observedModel, error: null, agentExitCode };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/* ─────────────────────────── runnable (CLI) entry ─────────────────────────── */
|
|
135
|
+
|
|
136
|
+
// True when this file is executed directly (`node run.mjs ...`), false when imported.
|
|
137
|
+
const isMain = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
|
|
138
|
+
|
|
139
|
+
if (isMain) {
|
|
140
|
+
// argv[2] is the AgentRunInput JSON (never contains secrets).
|
|
141
|
+
const input = JSON.parse(process.argv[2]);
|
|
142
|
+
|
|
143
|
+
// Always produce a RunnerResult, even if the runner itself throws, so the host
|
|
144
|
+
// always has a result file to read (node exit code stays 0 except on a truly
|
|
145
|
+
// unrecoverable crash before we can write).
|
|
146
|
+
let result;
|
|
147
|
+
try {
|
|
148
|
+
result = await runAgent(input);
|
|
149
|
+
} catch (e) {
|
|
150
|
+
result = {
|
|
151
|
+
ok: false,
|
|
152
|
+
output: '',
|
|
153
|
+
transcript: null,
|
|
154
|
+
observedModel: null,
|
|
155
|
+
error: e && e.message ? e.message : String(e),
|
|
156
|
+
agentExitCode: -1,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Source of truth: the result file the host reads back via sandbox.readFile.
|
|
161
|
+
try {
|
|
162
|
+
mkdirSync(dirname(input.resultPath), { recursive: true });
|
|
163
|
+
writeFileSync(input.resultPath, JSON.stringify(result));
|
|
164
|
+
} catch {
|
|
165
|
+
// If the file can't be written, the host falls back to the marker line below.
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// Fallback channel: a compact status line (no transcript — it can be huge).
|
|
169
|
+
process.stdout.write(
|
|
170
|
+
'__AGENT_RESULT__ ' +
|
|
171
|
+
JSON.stringify({
|
|
172
|
+
ok: result.ok,
|
|
173
|
+
observedModel: result.observedModel,
|
|
174
|
+
error: result.error,
|
|
175
|
+
agentExitCode: result.agentExitCode,
|
|
176
|
+
}) +
|
|
177
|
+
'\n'
|
|
178
|
+
);
|
|
179
|
+
|
|
180
|
+
// Exit 0: "the runner ran". Agent success/failure is conveyed via result.ok, not
|
|
181
|
+
// the node exit code (the host distinguishes the two).
|
|
182
|
+
process.exit(0);
|
|
183
|
+
}
|