@vercel/agent-eval 2.2.0 → 2.2.1

package/dist/lib/agents/eval-helper.mjs

@@ -27,7 +27,7 @@
  * these files before validation).
  */
 
-import { spawnSync } from 'node:child_process';
+import { spawn } from 'node:child_process';
 import { readFileSync, existsSync, mkdirSync } from 'node:fs';
 import { expect } from 'vitest';
 
@@ -129,12 +129,31 @@ function readJudgeConfig() {
   }
 }
 
+/** spawn + collect, resolving on exit. Never rejects: judge failures become verdicts. */
+function spawnCollect(cmd, args) {
+  return new Promise((resolve) => {
+    const child = spawn(cmd, args, { cwd: process.cwd(), env: process.env });
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (chunk) => (stdout += chunk));
+    child.stderr.on('data', (chunk) => (stderr += chunk));
+    child.on('error', (err) => resolve({ stdout, stderr: `${stderr}\n${err.message}`, status: -1 }));
+    child.on('close', (status) => resolve({ stdout, stderr, status }));
+  });
+}
+
 /**
  * Run one agentic judgment IN this sandbox by re-invoking the codegen agent's
- * runner. Blocking (spawnSync) — the test has nothing else to do while the judge
- * works, and the sandbox-level timeout bounds it. Returns {pass, score?, reason}.
+ * runner. Async ON PURPOSE, not as style: the vitest worker answers the main
+ * process over an RPC channel whose per-call timeout is hardcoded to 60s (birpc's
+ * DEFAULT_TIMEOUT — no vitest config or env var reaches it). A judge run takes
+ * however long the model takes, routinely past a minute, and the old spawnSync
+ * blocked the worker's event loop for all of it, so in-flight calls like
+ * onTaskUpdate timed out and vitest failed the file EVEN WHEN EVERY TEST PASSED.
+ * Keeping the loop alive while the judge works is the entire fix; the sandbox
+ * timeout still bounds the run. Returns {pass, score?, reason}.
  */
-function runJudge(subject, criterion, opts = {}) {
+async function runJudge(subject, criterion, opts = {}) {
   const id = nextId();
   mkdirSync(JUDGE_IO_DIR, { recursive: true });
   const verdictPath = `${JUDGE_IO_DIR}/${id}-verdict.json`;
@@ -156,12 +175,7 @@ function runJudge(subject, criterion, opts = {}) {
     extra: cfg.extra ?? undefined,
   };
 
-  const res = spawnSync('node', [runnerPath, JSON.stringify(input)], {
-    cwd: process.cwd(),
-    env: process.env,
-    encoding: 'utf8',
-    maxBuffer: 64 * 1024 * 1024,
-  });
+  const res = await spawnCollect('node', [runnerPath, JSON.stringify(input)]);
 
   // Prefer the verdict file the judge agent wrote; then the runner result's output
   // (the agent's final message); then raw stdout.
@@ -189,9 +203,12 @@ function subjectOf(received) {
   return received && typeof received === 'object' ? received.__judgeSubject : undefined;
 }
 
-// Sync matchers: spawnSync blocks, so these work whether or not you `await` them.
+// Async matchers — `await expect(...)`, as every example here shows. The await is
+// load-bearing: vitest does not track a custom matcher's promise, so an un-awaited
+// judge call floats free of its test — a failing verdict surfaces as an unhandled
+// rejection instead of failing the test, and a passing one is never even read.
 expect.extend({
-  toSatisfyCriterion(received, criterion) {
+  async toSatisfyCriterion(received, criterion) {
     const subject = subjectOf(received);
     if (!subject) {
       return {
@@ -200,7 +217,7 @@ expect.extend({
           'toSatisfyCriterion expects `environment` or `transcript` from @vercel/agent-eval/eval',
       };
     }
-    const v = runJudge(subject, criterion);
+    const v = await runJudge(subject, criterion);
     return {
       pass: v.pass,
       message: () =>
@@ -210,7 +227,7 @@ expect.extend({
     };
   },
 
-  toScoreAtLeast(received, criterion, threshold) {
+  async toScoreAtLeast(received, criterion, threshold) {
     const subject = subjectOf(received);
     if (!subject) {
       return {
@@ -219,7 +236,7 @@ expect.extend({
           'toScoreAtLeast expects `environment` or `transcript` from @vercel/agent-eval/eval',
       };
     }
-    const v = runJudge(subject, criterion, { numeric: true });
+    const v = await runJudge(subject, criterion, { numeric: true });
     const score = typeof v.score === 'number' ? v.score : v.pass ? 1 : 0;
     const pass = score >= threshold;
     return {

package/dist/lib/agents/shared.d.ts

@@ -43,6 +43,34 @@ export interface NeutralWorkspace {
     cwd: string;
     env: Record<string, string>;
 }
+/**
+ * Version installed when the workspace has no vitest of its own. Only a floor:
+ * anything recent satisfies the generated config, which uses nothing but
+ * `defineConfig`.
+ */
+export declare const FALLBACK_VITEST_VERSION = "^3";
+/**
+ * Guarantee the validation runner is resolvable from the workspace.
+ *
+ * Validation shells out to `npx vitest`, and the generated vitest.config.ts
+ * imports `vitest/config`, which Node resolves from the workspace. Fixtures
+ * normally supply vitest as a devDependency, but package.json belongs to the agent
+ * for the length of the run. An agent scaffolding into an empty directory often
+ * replaces that file outright rather than editing it, and its next `npm install`
+ * prunes vitest. npx then downloads vitest into its own cache, which does not
+ * satisfy the config's import, and the run dies at startup with "Cannot find
+ * module 'vitest/config'".
+ *
+ * That failure is indistinguishable from a bad result: the eval never executes, so
+ * the harness records a red for work it never graded. Installing vitest when it is
+ * missing keeps a fixture gradeable no matter what the agent did to the manifest.
+ *
+ * Nothing about this repair may show up as the agent's work: it runs before the
+ * git diff is captured, so `--no-save` keeps it out of package.json and
+ * `--no-package-lock` keeps it out of the lockfile, which is not gitignored and
+ * would otherwise be recorded as a file the agent wrote.
+ */
+export declare function ensureValidationRunner(sandbox: AnySandbox): Promise<void>;
 /**
  * Run validation scripts in the sandbox.
  */

package/dist/lib/agents/shared.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/shared.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,6CAA6C;AAC7C,KAAK,UAAU,GAAG,cAAc,GAAG,oBAAoB,CAAC;AAExD;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,mBAAmB,CAAC;AAEvD,mDAAmD;AACnD,eAAO,MAAM,uBAAuB,gCAA2C,CAAC;AAMhF,oFAAoF;AACpF,eAAO,MAAM,gBAAgB,mCAA8C,CAAC;AAE5E,oFAAoF;AACpF,eAAO,MAAM,qBAAqB,kCAA6C,CAAC;AAEhF,6EAA6E;AAC7E,eAAO,MAAM,iBAAiB,qCAAgD,CAAC;AAE/E;oFACoF;AACpF,eAAO,MAAM,iBAAiB,iCAA4C,CAAC;AAE3E;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,eAAe,EAAE,MAAM,MAAM,GAAG,MAAM,GAAG,SAAS,CAEpF;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC7B;AAgCD;;GAEG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,UAAU,EACnB,OAAO,EAAE,MAAM,EAAE,EACjB,UAAU,GAAE,cAAyB,EAIrC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC3B,OAAO,CAAC,iBAAiB,CAAC,CAqC5B;AAED,wBAAsB,uBAAuB,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA0B5F;AAED,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAYzE;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC;IAAE,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,YAAY,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAuC7E;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CA4B3E;AAED;;;;;GAKG;AACH,wBAAsB,uBAAuB,CAC3C,OAAO,EAAE,UAAU,EACnB,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,SAAS,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED;;GAEG;AACH,eAAO,MAAM,UAAU;;;;CAIb,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,gBAAgB;;CAEnB,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,aAAa;;;CAGhB,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,aAAa;;CAEhB,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,aAAa;;CAEhB,CAAC"}
+{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/shared.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,6CAA6C;AAC7C,KAAK,UAAU,GAAG,cAAc,GAAG,oBAAoB,CAAC;AAExD;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,mBAAmB,CAAC;AAEvD,mDAAmD;AACnD,eAAO,MAAM,uBAAuB,gCAA2C,CAAC;AAMhF,oFAAoF;AACpF,eAAO,MAAM,gBAAgB,mCAA8C,CAAC;AAE5E,oFAAoF;AACpF,eAAO,MAAM,qBAAqB,kCAA6C,CAAC;AAEhF,6EAA6E;AAC7E,eAAO,MAAM,iBAAiB,qCAAgD,CAAC;AAE/E;oFACoF;AACpF,eAAO,MAAM,iBAAiB,iCAA4C,CAAC;AAE3E;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,eAAe,EAAE,MAAM,MAAM,GAAG,MAAM,GAAG,SAAS,CAEpF;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC7B;AAgCD;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,OAAO,CAAC;AAE5C;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAc/E;AAED;;GAEG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,UAAU,EACnB,OAAO,EAAE,MAAM,EAAE,EACjB,UAAU,GAAE,cAAyB,EAIrC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC3B,OAAO,CAAC,iBAAiB,CAAC,CAwC5B;AAED,wBAAsB,uBAAuB,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA0B5F;AAED,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAYzE;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC;IAAE,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,YAAY,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAuC7E;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CA4B3E;AAED;;;;;GAKG;AACH,wBAAsB,uBAAuB,CAC3C,OAAO,EAAE,UAAU,EACnB,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,SAAS,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED;;GAEG;AACH,eAAO,MAAM,UAAU;;;;CAIb,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,gBAAgB;;CAEnB,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,aAAa;;;CAGhB,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,aAAa;;CAEhB,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,aAAa;;CAEhB,CAAC"}

package/dist/lib/agents/shared.js

@@ -58,6 +58,47 @@ async function detectEvalFile(sandbox) {
     // Default to EVAL.ts (will fail later if it doesn't exist)
     return 'EVAL.ts';
 }
+/**
+ * Version installed when the workspace has no vitest of its own. Only a floor:
+ * anything recent satisfies the generated config, which uses nothing but
+ * `defineConfig`.
+ */
+export const FALLBACK_VITEST_VERSION = '^3';
+/**
+ * Guarantee the validation runner is resolvable from the workspace.
+ *
+ * Validation shells out to `npx vitest`, and the generated vitest.config.ts
+ * imports `vitest/config`, which Node resolves from the workspace. Fixtures
+ * normally supply vitest as a devDependency, but package.json belongs to the agent
+ * for the length of the run. An agent scaffolding into an empty directory often
+ * replaces that file outright rather than editing it, and its next `npm install`
+ * prunes vitest. npx then downloads vitest into its own cache, which does not
+ * satisfy the config's import, and the run dies at startup with "Cannot find
+ * module 'vitest/config'".
+ *
+ * That failure is indistinguishable from a bad result: the eval never executes, so
+ * the harness records a red for work it never graded. Installing vitest when it is
+ * missing keeps a fixture gradeable no matter what the agent did to the manifest.
+ *
+ * Nothing about this repair may show up as the agent's work: it runs before the
+ * git diff is captured, so `--no-save` keeps it out of package.json and
+ * `--no-package-lock` keeps it out of the lockfile, which is not gitignored and
+ * would otherwise be recorded as a file the agent wrote.
+ */
+export async function ensureValidationRunner(sandbox) {
+    const present = await sandbox.runShell('test -e node_modules/vitest/package.json');
+    if (present.exitCode === 0) {
+        return;
+    }
+    await sandbox.runCommand('npm', [
+        'install',
+        '--no-save',
+        '--no-package-lock',
+        '--no-audit',
+        '--no-fund',
+        `vitest@${FALLBACK_VITEST_VERSION}`,
+    ]);
+}
 /**
  * Run validation scripts in the sandbox.
  */
@@ -73,6 +114,8 @@ env) {
     if (validation === 'vitest') {
         // Detect which eval file exists (EVAL.ts or EVAL.tsx)
         const evalFile = await detectEvalFile(sandbox);
+        // The agent may have removed vitest from the project; put it back first.
+        await ensureValidationRunner(sandbox);
         // Always run vitest for the eval file (explicitly specify the file)
         const testResult = await sandbox.runCommand('npx', ['vitest', 'run', evalFile], env ? { env } : undefined);
         results.test = {

package/dist/lib/agents/shared.js.map

@@ -1 +1 @@
-{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../src/lib/agents/shared.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMnD;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,gBAAgB,CAAC;AAEvD,mDAAmD;AACnD,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,sBAAsB,eAAe,CAAC;AAEhF,kFAAkF;AAClF,gFAAgF;AAChF,oDAAoD;AAEpD,oFAAoF;AACpF,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,sBAAsB,kBAAkB,CAAC;AAE5E,oFAAoF;AACpF,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,sBAAsB,iBAAiB,CAAC;AAEhF,6EAA6E;AAC7E,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,sBAAsB,oBAAoB,CAAC;AAE/E;oFACoF;AACpF,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,sBAAsB,gBAAgB,CAAC;AAE3E;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,eAA6B;IAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;AACzE,CAAC;AAgBD;;;;GAIG;AACH,KAAK,UAAU,cAAc,CAAC,OAAmB;IAC/C,IAAI,CAAC;QACH,iEAAiE;QACjE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACjD,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAE/D,sDAAsD;YACtD,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC/B,OAAO,UAAU,CAAC;YACpB,CAAC;YAED,oBAAoB;YACpB,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC9B,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gBAAgB;IAClB,CAAC;IAED,2DAA2D;IAC3D,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAmB,EACnB,OAAiB,EACjB,aAA6B,QAAQ;AACrC,8EAA8E;AAC9E,gFAAgF;AAChF,gFAAgF;AAChF,GAA4B;IAE5B,MAAM,OAAO,GAAsB;QACjC,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,EAAE;KACZ,CAAC;IAEF,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;QAC5B,sDAAsD;QACtD,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;QAE/C,oEAAoE;QACpE,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC3G,OAAO,CAAC,IAAI,GAAG;YACb,OAAO,EAAE,UAAU,CAAC,QAAQ,KAAK,CAAC;YAClC,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM;SAC9C,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAC1B,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QACtE,MAAM,MAAM,GAAiB;YAC3B,OAAO,EAAE,YAAY,CAAC,QAAQ,KAAK,CAAC;YACpC,MAAM,EAAE,YAAY,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM;SAClD,CAAC;QAEF,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;QAEjC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAAmB;IAC/D,MAAM,UAAU,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACrD,MAAM,uBAAuB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAE9D,MAAM,OAAO,CAAC,QAAQ,CAAC,gEAAgE,CAAC,CAAC;IAEzF,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAClD,OAAO,EAAE,GAAG,EAAE,uBAAuB,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;IAC3D,CAAC;IAED,MAAM,oBAAoB,GAAG,YAAY,CAAC;IAC1C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CACvC;QACE,eAAe,oBAAoB,EAAE;QACrC,iBAAiB,oBAAoB,EAAE;QACvC,gBAAgB,oBAAoB,GAAG;QACvC,qCAAqC,oBAAoB,EAAE;KAC5D,CAAC,IAAI,CAAC,MAAM,CAAC,CACf,CAAC;IACF,IAAI,UAAU,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChG,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,OAAO,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,CAAC;IAClD,OAAO,EAAE,GAAG,EAAE,oBAAoB,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;AACxD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAmB;IACxD,MAAM,OAAO,CAAC,UAAU,CAAC;QACvB,8EAA8E;QAC9E,6EAA6E;QAC7E,YAAY,EAAE,kCAAkC;KACjD,CAAC,CAAC;IAEH,4HAA4H;IAC5H,sBAAsB;IACtB,MAAM,OAAO,CAAC,QAAQ,CACpB,oIAAoI,CACrI,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,OAAmB;IAEnB,MAAM,cAAc,GAA2B,EAAE,CAAC;IAClD,MAAM,YAAY,GAAa,EAAE,CAAC;IAElC,IAAI,CAAC;QACH,+DAA+D;QAC/D,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,0CAA0C,CAAC,CAAC;QAEtF,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM;aAC5B,IAAI,EAAE;aACN,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEjC,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAExB,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;gBACnB,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC;oBACH,oEAAoE;oBACpE,oEAAoE;oBACpE,mEAAmE;oBACnE,oCAAoC;oBACpC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;oBACvD,cAAc,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;gBACrC,CAAC;gBAAC,MAAM,CAAC;oBACP,wBAAwB;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,yCAAyC;IAC3C,CAAC;IAED,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAmB;IAC1D,gCAAgC;IAChC,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAE/C,yEAAyE;IACzE,gFAAgF;IAChF,sEAAsE;IACtE,MAAM,UAAU,GAAG,GAAG,OAAO,CAAC,mBAAmB,EAAE,IAAI,gBAAgB,EAAE,CAAC;IAE1E,MAAM,OAAO,CAAC,UAAU,CAAC;QACvB,kBAAkB,EAAE;;;;iBAIP,QAAQ;;;;;;oBAML,UAAU;;;2CAGa,UAAU;;;CAGpD;KACE,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,OAAmB,EACnB,aAAiC,EACjC,SAAiB,EACjB,KAAc;IAEd,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,aAAa;YAC9B,CAAC,CAAC,eAAe,CAAC,aAAa,EAAE,SAAS,EAAE,KAAK,CAAC;YAClD,CAAC,CAAC,IAAI,CAAC;QAET,MAAM,OAAO,GAAG;YACd,IAAI,EAAE,UAAU,EAAE,OAAO,IAAI,IAAI;SAClC,CAAC;QAEF,MAAM,OAAO,CAAC,UAAU,CAAC;YACvB,CAAC,uBAAuB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;SAC5D,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;IAChE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,OAAO,EAAE,8BAA8B;IACvC,aAAa,EAAE,iCAAiC;IAChD,YAAY,EAAE,oBAAoB;CAC1B,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,YAAY,EAAE,mBAAmB;CACzB,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,OAAO,EAAE,2BAA2B;IACpC,YAAY,EAAE,gBAAgB;CACtB,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,YAAY,EAAE,gBAAgB;CACtB,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,YAAY,EAAE,gBAAgB;CACtB,CAAC"}
+{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../src/lib/agents/shared.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMnD;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,gBAAgB,CAAC;AAEvD,mDAAmD;AACnD,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,sBAAsB,eAAe,CAAC;AAEhF,kFAAkF;AAClF,gFAAgF;AAChF,oDAAoD;AAEpD,oFAAoF;AACpF,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,sBAAsB,kBAAkB,CAAC;AAE5E,oFAAoF;AACpF,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,sBAAsB,iBAAiB,CAAC;AAEhF,6EAA6E;AAC7E,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,sBAAsB,oBAAoB,CAAC;AAE/E;oFACoF;AACpF,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,sBAAsB,gBAAgB,CAAC;AAE3E;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,eAA6B;IAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;AACzE,CAAC;AAgBD;;;;GAIG;AACH,KAAK,UAAU,cAAc,CAAC,OAAmB;IAC/C,IAAI,CAAC;QACH,iEAAiE;QACjE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACjD,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAE/D,sDAAsD;YACtD,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC/B,OAAO,UAAU,CAAC;YACpB,CAAC;YAED,oBAAoB;YACpB,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC9B,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gBAAgB;IAClB,CAAC;IAED,2DAA2D;IAC3D,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAE5C;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,OAAmB;IAC9D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,0CAA0C,CAAC,CAAC;IACnF,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO;IACT,CAAC;IAED,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE;QAC9B,SAAS;QACT,WAAW;QACX,mBAAmB;QACnB,YAAY;QACZ,WAAW;QACX,UAAU,uBAAuB,EAAE;KACpC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAmB,EACnB,OAAiB,EACjB,aAA6B,QAAQ;AACrC,8EAA8E;AAC9E,gFAAgF;AAChF,gFAAgF;AAChF,GAA4B;IAE5B,MAAM,OAAO,GAAsB;QACjC,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,EAAE;KACZ,CAAC;IAEF,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;QAC5B,sDAAsD;QACtD,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;QAE/C,yEAAyE;QACzE,MAAM,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAEtC,oEAAoE;QACpE,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC3G,OAAO,CAAC,IAAI,GAAG;YACb,OAAO,EAAE,UAAU,CAAC,QAAQ,KAAK,CAAC;YAClC,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM;SAC9C,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAC1B,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QACtE,MAAM,MAAM,GAAiB;YAC3B,OAAO,EAAE,YAAY,CAAC,QAAQ,KAAK,CAAC;YACpC,MAAM,EAAE,YAAY,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM;SAClD,CAAC;QAEF,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;QAEjC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAAmB;IAC/D,MAAM,UAAU,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACrD,MAAM,uBAAuB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAE9D,MAAM,OAAO,CAAC,QAAQ,CAAC,gEAAgE,CAAC,CAAC;IAEzF,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAClD,OAAO,EAAE,GAAG,EAAE,uBAAuB,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;IAC3D,CAAC;IAED,MAAM,oBAAoB,GAAG,YAAY,CAAC;IAC1C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CACvC;QACE,eAAe,oBAAoB,EAAE;QACrC,iBAAiB,oBAAoB,EAAE;QACvC,gBAAgB,oBAAoB,GAAG;QACvC,qCAAqC,oBAAoB,EAAE;KAC5D,CAAC,IAAI,CAAC,MAAM,CAAC,CACf,CAAC;IACF,IAAI,UAAU,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChG,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,OAAO,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,CAAC;IAClD,OAAO,EAAE,GAAG,EAAE,oBAAoB,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;AACxD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAmB;IACxD,MAAM,OAAO,CAAC,UAAU,CAAC;QACvB,8EAA8E;QAC9E,6EAA6E;QAC7E,YAAY,EAAE,kCAAkC;KACjD,CAAC,CAAC;IAEH,4HAA4H;IAC5H,sBAAsB;IACtB,MAAM,OAAO,CAAC,QAAQ,CACpB,oIAAoI,CACrI,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,OAAmB;IAEnB,MAAM,cAAc,GAA2B,EAAE,CAAC;IAClD,MAAM,YAAY,GAAa,EAAE,CAAC;IAElC,IAAI,CAAC;QACH,+DAA+D;QAC/D,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,0CAA0C,CAAC,CAAC;QAEtF,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM;aAC5B,IAAI,EAAE;aACN,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEjC,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAExB,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;gBACnB,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC;oBACH,oEAAoE;oBACpE,oEAAoE;oBACpE,mEAAmE;oBACnE,oCAAoC;oBACpC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;oBACvD,cAAc,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;gBACrC,CAAC;gBAAC,MAAM,CAAC;oBACP,wBAAwB;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,yCAAyC;IAC3C,CAAC;IAED,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAmB;IAC1D,gCAAgC;IAChC,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAE/C,yEAAyE;IACzE,gFAAgF;IAChF,sEAAsE;IACtE,MAAM,UAAU,GAAG,GAAG,OAAO,CAAC,mBAAmB,EAAE,IAAI,gBAAgB,EAAE,CAAC;IAE1E,MAAM,OAAO,CAAC,UAAU,CAAC;QACvB,kBAAkB,EAAE;;;;iBAIP,QAAQ;;;;;;oBAML,UAAU;;;2CAGa,UAAU;;;CAGpD;KACE,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,OAAmB,EACnB,aAAiC,EACjC,SAAiB,EACjB,KAAc;IAEd,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,aAAa;YAC9B,CAAC,CAAC,eAAe,CAAC,aAAa,EAAE,SAAS,EAAE,KAAK,CAAC;YAClD,CAAC,CAAC,IAAI,CAAC;QAET,MAAM,OAAO,GAAG;YACd,IAAI,EAAE,UAAU,EAAE,OAAO,IAAI,IAAI;SAClC,CAAC;QAEF,MAAM,OAAO,CAAC,UAAU,CAAC;YACvB,CAAC,uBAAuB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;SAC5D,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;IAChE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,OAAO,EAAE,8BAA8B;IACvC,aAAa,EAAE,iCAAiC;IAChD,YAAY,EAAE,oBAAoB;CAC1B,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,YAAY,EAAE,mBAAmB;CACzB,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,OAAO,EAAE,2BAA2B;IACpC,YAAY,EAAE,gBAAgB;CACtB,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,YAAY,EAAE,gBAAgB;CACtB,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,YAAY,EAAE,gBAAgB;CACtB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vercel/agent-eval",
-  "version": "2.2.0",
+  "version": "2.2.1",
   "description": "Framework for testing AI coding agents in isolated sandboxes",
   "repository": {
     "type": "git",