@vercel/agent-eval 2.1.0 → 2.2.1

package/dist/index.d.ts

@@ -26,5 +26,5 @@ export { Dashboard, createConsoleProgressHandler } from './lib/dashboard.js';
 export type { InitOptions } from './lib/init.js';
 export { initProject, getPostInitInstructions } from './lib/init.js';
 export type { ToolName, TranscriptEvent, WebFetchInfo, FileOperationInfo, ShellCommandInfo, TranscriptSummary, Transcript, ParseableAgent, } from './lib/o11y/index.js';
-export { parseTranscript, parseTranscriptSummary, loadTranscript, SUPPORTED_AGENTS, parseClaudeCodeTranscript, parseCodexTranscript, parseOpenCodeTranscript, } from './lib/o11y/index.js';
+export { parseTranscript, parseTranscriptSummary, loadTranscript, SUPPORTED_AGENTS, parseClaudeCodeTranscript, parseCodexTranscript, parseOpenCodeTranscript, parseFxTranscript, } from './lib/o11y/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,YAAY,EACV,SAAS,EACT,gBAAgB,EAChB,SAAS,EACT,WAAW,EACX,UAAU,EACV,cAAc,EACd,WAAW,EACX,OAAO,EACP,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,gBAAgB,EAChB,wBAAwB,EACxB,WAAW,EACX,aAAa,EACb,WAAW,EACX,WAAW,EACX,aAAa,EACb,iBAAiB,EACjB,WAAW,EACX,cAAc,GACf,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAG3F,OAAO,EACL,eAAe,EACf,cAAc,EACd,aAAa,EACb,UAAU,EACV,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,sBAAsB,EACtB,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,EACnB,WAAW,EACX,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,cAAc,EACd,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,cAAc,EACd,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAG/D,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAG7E,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAGzF,YAAY,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAG5E,YAAY,EAAE,kBAAkB,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAC7F,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,kBAAkB,EAAE,oCAAoC,EAAE,MAAM,sBAAsB,CAAC;AAGhG,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,cAAc,EACd,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAGlD,YAAY,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAGjF,OAAO,EAAE,SAAS,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAG7E,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAGrE,YAAY,EACV,QAAQ,EACR,eAAe,EACf,YAAY,EACZ,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,UAAU,EACV,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,yBAAyB,EACzB,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,YAAY,EACV,SAAS,EACT,gBAAgB,EAChB,SAAS,EACT,WAAW,EACX,UAAU,EACV,cAAc,EACd,WAAW,EACX,OAAO,EACP,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,gBAAgB,EAChB,wBAAwB,EACxB,WAAW,EACX,aAAa,EACb,WAAW,EACX,WAAW,EACX,aAAa,EACb,iBAAiB,EACjB,WAAW,EACX,cAAc,GACf,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAG3F,OAAO,EACL,eAAe,EACf,cAAc,EACd,aAAa,EACb,UAAU,EACV,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,sBAAsB,EACtB,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,EACnB,WAAW,EACX,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,cAAc,EACd,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,cAAc,EACd,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAG/D,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAG7E,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAGzF,YAAY,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAG5E,YAAY,EAAE,kBAAkB,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAC7F,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,kBAAkB,EAAE,oCAAoC,EAAE,MAAM,sBAAsB,CAAC;AAGhG,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,cAAc,EACd,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAGlD,YAAY,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAGjF,OAAO,EAAE,SAAS,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAG7E,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAGrE,YAAY,EACV,QAAQ,EACR,eAAe,EACf,YAAY,EACZ,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,UAAU,EACV,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,yBAAyB,EACzB,oBAAoB,EACpB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,qBAAqB,CAAC"}

package/dist/index.js

@@ -25,5 +25,5 @@ export { runExperiment, runSingleEval, StartRateLimiter } from './lib/runner.js'
 // Re-export dashboard utilities
 export { Dashboard, createConsoleProgressHandler } from './lib/dashboard.js';
 export { initProject, getPostInitInstructions } from './lib/init.js';
-export { parseTranscript, parseTranscriptSummary, loadTranscript, SUPPORTED_AGENTS, parseClaudeCodeTranscript, parseCodexTranscript, parseOpenCodeTranscript, } from './lib/o11y/index.js';
+export { parseTranscript, parseTranscriptSummary, loadTranscript, SUPPORTED_AGENTS, parseClaudeCodeTranscript, parseCodexTranscript, parseOpenCodeTranscript, parseFxTranscript, } from './lib/o11y/index.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA2BH,sBAAsB;AACtB,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAE3F,6BAA6B;AAC7B,OAAO,EACL,eAAe,EACf,cAAc,EACd,aAAa,EACb,UAAU,EACV,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AAEzB,8BAA8B;AAC9B,OAAO,EACL,sBAAsB,EACtB,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,EACnB,WAAW,EACX,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAU1B,OAAO,EACL,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,cAAc,EACd,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAK/D,yCAAyC;AACzC,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAIzF,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAI5E,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,kBAAkB,CAAC;AAE1B,2BAA2B;AAC3B,OAAO,EAAE,kBAAkB,EAAE,oCAAoC,EAAE,MAAM,sBAAsB,CAAC;AAEhG,uBAAuB;AACvB,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,cAAc,EACd,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAE7B,yBAAyB;AACzB,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAIlD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAEjF,gCAAgC;AAChC,OAAO,EAAE,SAAS,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAI7E,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAarE,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,yBAAyB,EACzB,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,qBAAqB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA2BH,sBAAsB;AACtB,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAE3F,6BAA6B;AAC7B,OAAO,EACL,eAAe,EACf,cAAc,EACd,aAAa,EACb,UAAU,EACV,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AAEzB,8BAA8B;AAC9B,OAAO,EACL,sBAAsB,EACtB,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,EACnB,WAAW,EACX,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAU1B,OAAO,EACL,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,cAAc,EACd,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAK/D,yCAAyC;AACzC,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAIzF,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAI5E,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,kBAAkB,CAAC;AAE1B,2BAA2B;AAC3B,OAAO,EAAE,kBAAkB,EAAE,oCAAoC,EAAE,MAAM,sBAAsB,CAAC;AAEhG,uBAAuB;AACvB,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,cAAc,EACd,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAE7B,yBAAyB;AACzB,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAIlD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAEjF,gCAAgC;AAChC,OAAO,EAAE,SAAS,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAI7E,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAarE,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,yBAAyB,EACzB,oBAAoB,EACpB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,qBAAqB,CAAC"}

package/dist/lib/agents/eval-helper.mjs

@@ -27,7 +27,7 @@
  * these files before validation).
  */
 
-import { spawnSync } from 'node:child_process';
+import { spawn } from 'node:child_process';
 import { readFileSync, existsSync, mkdirSync } from 'node:fs';
 import { expect } from 'vitest';
 
@@ -129,12 +129,31 @@ function readJudgeConfig() {
   }
 }
 
+/** spawn + collect, resolving on exit. Never rejects: judge failures become verdicts. */
+function spawnCollect(cmd, args) {
+  return new Promise((resolve) => {
+    const child = spawn(cmd, args, { cwd: process.cwd(), env: process.env });
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (chunk) => (stdout += chunk));
+    child.stderr.on('data', (chunk) => (stderr += chunk));
+    child.on('error', (err) => resolve({ stdout, stderr: `${stderr}\n${err.message}`, status: -1 }));
+    child.on('close', (status) => resolve({ stdout, stderr, status }));
+  });
+}
+
 /**
  * Run one agentic judgment IN this sandbox by re-invoking the codegen agent's
- * runner. Blocking (spawnSync) — the test has nothing else to do while the judge
- * works, and the sandbox-level timeout bounds it. Returns {pass, score?, reason}.
+ * runner. Async ON PURPOSE, not as style: the vitest worker answers the main
+ * process over an RPC channel whose per-call timeout is hardcoded to 60s (birpc's
+ * DEFAULT_TIMEOUT — no vitest config or env var reaches it). A judge run takes
+ * however long the model takes, routinely past a minute, and the old spawnSync
+ * blocked the worker's event loop for all of it, so in-flight calls like
+ * onTaskUpdate timed out and vitest failed the file EVEN WHEN EVERY TEST PASSED.
+ * Keeping the loop alive while the judge works is the entire fix; the sandbox
+ * timeout still bounds the run. Returns {pass, score?, reason}.
  */
-function runJudge(subject, criterion, opts = {}) {
+async function runJudge(subject, criterion, opts = {}) {
   const id = nextId();
   mkdirSync(JUDGE_IO_DIR, { recursive: true });
   const verdictPath = `${JUDGE_IO_DIR}/${id}-verdict.json`;
@@ -156,12 +175,7 @@ function runJudge(subject, criterion, opts = {}) {
     extra: cfg.extra ?? undefined,
   };
 
-  const res = spawnSync('node', [runnerPath, JSON.stringify(input)], {
-    cwd: process.cwd(),
-    env: process.env,
-    encoding: 'utf8',
-    maxBuffer: 64 * 1024 * 1024,
-  });
+  const res = await spawnCollect('node', [runnerPath, JSON.stringify(input)]);
 
   // Prefer the verdict file the judge agent wrote; then the runner result's output
   // (the agent's final message); then raw stdout.
@@ -189,9 +203,12 @@ function subjectOf(received) {
   return received && typeof received === 'object' ? received.__judgeSubject : undefined;
 }
 
-// Sync matchers: spawnSync blocks, so these work whether or not you `await` them.
+// Async matchers — `await expect(...)`, as every example here shows. The await is
+// load-bearing: vitest does not track a custom matcher's promise, so an un-awaited
+// judge call floats free of its test — a failing verdict surfaces as an unhandled
+// rejection instead of failing the test, and a passing one is never even read.
 expect.extend({
-  toSatisfyCriterion(received, criterion) {
+  async toSatisfyCriterion(received, criterion) {
     const subject = subjectOf(received);
     if (!subject) {
       return {
@@ -200,7 +217,7 @@ expect.extend({
           'toSatisfyCriterion expects `environment` or `transcript` from @vercel/agent-eval/eval',
       };
     }
-    const v = runJudge(subject, criterion);
+    const v = await runJudge(subject, criterion);
     return {
       pass: v.pass,
       message: () =>
@@ -210,7 +227,7 @@ expect.extend({
     };
   },
 
-  toScoreAtLeast(received, criterion, threshold) {
+  async toScoreAtLeast(received, criterion, threshold) {
     const subject = subjectOf(received);
     if (!subject) {
       return {
@@ -219,7 +236,7 @@ expect.extend({
           'toScoreAtLeast expects `environment` or `transcript` from @vercel/agent-eval/eval',
       };
     }
-    const v = runJudge(subject, criterion, { numeric: true });
+    const v = await runJudge(subject, criterion, { numeric: true });
     const score = typeof v.score === 'number' ? v.score : v.pass ? 1 : 0;
     const pass = score >= threshold;
     return {

package/dist/lib/agents/fx/agent.d.ts

@@ -0,0 +1,15 @@
+/**
+ * fx agent definition for research-enabled runs through Vercel AI Gateway.
+ */
+import type { Agent } from '../types.js';
+import type { AgentDefinition } from '../plugin/contract.js';
+export declare const FX_VERSION = "0.0.5";
+export declare const FX_RELEASE_SHA256: {
+    readonly x64: "d5639d173267774aa8228a474baf619a7076ac41a91023915007c865143429b1";
+    readonly arm64: "8bbcde6a41256c4fac4e0a022291cf02740419e27afabde3b8f45e7a4e393edb";
+};
+/** Build the zero-dependency installer executed inside the Linux sandbox. */
+export declare function buildFxInstallScript(): string;
+export declare function createFxDefinition(): AgentDefinition;
+export declare function createFxAgent(): Agent;
+//# sourceMappingURL=agent.d.ts.map

package/dist/lib/agents/fx/agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../../../src/lib/agents/fx/agent.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,KAAK,EAAmB,MAAM,aAAa,CAAC;AAG1D,OAAO,KAAK,EAAE,eAAe,EAAe,MAAM,uBAAuB,CAAC;AAG1E,eAAO,MAAM,UAAU,UAAU,CAAC;AAElC,eAAO,MAAM,iBAAiB;;;CAGpB,CAAC;AAIX,6EAA6E;AAC7E,wBAAgB,oBAAoB,IAAI,MAAM,CAiE7C;AAED,wBAAgB,kBAAkB,IAAI,eAAe,CAgDpD;AAED,wBAAgB,aAAa,IAAI,KAAK,CAYrC"}

package/dist/lib/agents/fx/agent.js

@@ -0,0 +1,137 @@
+/**
+ * fx agent definition for research-enabled runs through Vercel AI Gateway.
+ */
+import { fileURLToPath } from 'node:url';
+import { AI_GATEWAY } from '../shared.js';
+import { runWithDefinition } from '../plugin/orchestrator.js';
+export const FX_VERSION = '0.0.5';
+export const FX_RELEASE_SHA256 = {
+    x64: 'd5639d173267774aa8228a474baf619a7076ac41a91023915007c865143429b1',
+    arm64: '8bbcde6a41256c4fac4e0a022291cf02740419e27afabde3b8f45e7a4e393edb',
+};
+const FX_RUNTIME_PROTOCOL = 'ask-json-session-v1';
+/** Build the zero-dependency installer executed inside the Linux sandbox. */
+export function buildFxInstallScript() {
+    return `
+import { createHash } from 'node:crypto';
+import { chmodSync, mkdirSync, unlinkSync, writeFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { spawnSync } from 'node:child_process';
+
+const version = ${JSON.stringify(FX_VERSION)};
+const checksums = ${JSON.stringify(FX_RELEASE_SHA256)};
+if (process.platform !== 'linux' || !(process.arch in checksums)) {
+  throw new Error(\`Unsupported fx sandbox platform: \${process.platform}/\${process.arch}\`);
+}
+
+const releaseArch = process.arch === 'x64' ? 'x86_64' : 'aarch64';
+const asset = \`fx-linux-\${releaseArch}.tar.gz\`;
+const url = \`https://github.com/vercel-labs/fx/releases/download/v\${version}/\${asset}\`;
+const installDir = resolve('__agent_eval__/bin');
+const archivePath = join(installDir, asset);
+const binaryPath = join(installDir, 'fx');
+
+mkdirSync(installDir, { recursive: true });
+const maxArchiveBytes = 32 * 1024 * 1024;
+const controller = new AbortController();
+const timeout = setTimeout(() => controller.abort(), 60_000);
+let archive;
+try {
+  const response = await fetch(url, { signal: controller.signal });
+  if (!response.ok) throw new Error(\`fx download failed: HTTP \${response.status}\`);
+  const contentLength = Number(response.headers.get('content-length'));
+  if (Number.isFinite(contentLength) && contentLength > maxArchiveBytes) {
+    throw new Error('fx archive exceeds 32 MB');
+  }
+  if (!response.body) throw new Error('fx download returned no body');
+
+  const chunks = [];
+  let size = 0;
+  for await (const chunk of response.body) {
+    const bytes = Buffer.from(chunk);
+    size += bytes.length;
+    if (size > maxArchiveBytes) {
+      controller.abort();
+      throw new Error('fx archive exceeds 32 MB');
+    }
+    chunks.push(bytes);
+  }
+  archive = Buffer.concat(chunks, size);
+} finally {
+  clearTimeout(timeout);
+}
+
+const actual = createHash('sha256').update(archive).digest('hex');
+const expected = checksums[process.arch];
+if (actual !== expected) throw new Error(\`fx checksum mismatch: expected \${expected}, got \${actual}\`);
+
+writeFileSync(archivePath, archive);
+const extracted = spawnSync('tar', ['-xzf', archivePath, '-C', installDir, 'fx'], { encoding: 'utf8' });
+unlinkSync(archivePath);
+if (extracted.status !== 0) throw new Error(\`fx extraction failed: \${extracted.stderr || extracted.error?.message || 'unknown error'}\`);
+chmodSync(binaryPath, 0o755);
+
+const installed = spawnSync(binaryPath, ['--version'], { encoding: 'utf8' });
+if (installed.status !== 0 || installed.stdout.trim() !== version) {
+  throw new Error(\`fx version check failed: \${installed.stdout || installed.stderr || installed.error?.message || 'unknown error'}\`);
+}
+`;
+}
+export function createFxDefinition() {
+    return {
+        name: 'vercel-ai-gateway/fx',
+        displayName: 'fx (Vercel AI Gateway)',
+        defaultModel: 'zai/glm-5.2',
+        o11yAgentName: 'vercel-ai-gateway/fx',
+        bundledSkillsControl: 'not-applicable',
+        requiresWebResearch: true,
+        supportsCrossAgentJudge: false,
+        runnerPath: fileURLToPath(new URL('./run.mjs', import.meta.url)),
+        getApiKeyEnvVar() {
+            return AI_GATEWAY.apiKeyEnvVar;
+        },
+        install(_options) {
+            return [
+                {
+                    kind: 'command',
+                    cmd: 'npm',
+                    args: ['install'],
+                    retryOnce: true,
+                    errorPrefix: 'npm install failed',
+                    errorBody: 'last10',
+                },
+                {
+                    kind: 'command',
+                    cmd: 'node',
+                    args: ['--input-type=module', '--eval', buildFxInstallScript()],
+                    errorPrefix: 'fx install failed',
+                    errorBody: 'stderr',
+                },
+            ];
+        },
+        configFiles: () => [],
+        authEnv(options) {
+            return { [AI_GATEWAY.apiKeyEnvVar]: options.apiKey };
+        },
+        fingerprintExtra() {
+            return {
+                fxVersion: FX_VERSION,
+                fxRuntimeProtocol: FX_RUNTIME_PROTOCOL,
+            };
+        },
+    };
+}
+export function createFxAgent() {
+    const definition = createFxDefinition();
+    return {
+        name: definition.name,
+        displayName: definition.displayName,
+        getApiKeyEnvVar: definition.getApiKeyEnvVar,
+        getDefaultModel() {
+            return definition.defaultModel;
+        },
+        run: (fixturePath, options) => runWithDefinition(definition, fixturePath, options),
+        definition,
+    };
+}
+//# sourceMappingURL=agent.js.map

package/dist/lib/agents/fx/agent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent.js","sourceRoot":"","sources":["../../../../src/lib/agents/fx/agent.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,MAAM,CAAC,MAAM,UAAU,GAAG,OAAO,CAAC;AAElC,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,GAAG,EAAE,kEAAkE;IACvE,KAAK,EAAE,kEAAkE;CACjE,CAAC;AAEX,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;AAElD,6EAA6E;AAC7E,MAAM,UAAU,oBAAoB;IAClC,OAAO;;;;;;kBAMS,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC;oBACxB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwDpD,CAAC;AACF,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,OAAO;QACL,IAAI,EAAE,sBAAsB;QAC5B,WAAW,EAAE,wBAAwB;QACrC,YAAY,EAAE,aAAa;QAC3B,aAAa,EAAE,sBAAsB;QACrC,oBAAoB,EAAE,gBAAgB;QACtC,mBAAmB,EAAE,IAAI;QACzB,uBAAuB,EAAE,KAAK;QAC9B,UAAU,EAAE,aAAa,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEhE,eAAe;YACb,OAAO,UAAU,CAAC,YAAY,CAAC;QACjC,CAAC;QAED,OAAO,CAAC,QAAyB;YAC/B,OAAO;gBACL;oBACE,IAAI,EAAE,SAAS;oBACf,GAAG,EAAE,KAAK;oBACV,IAAI,EAAE,CAAC,SAAS,CAAC;oBACjB,SAAS,EAAE,IAAI;oBACf,WAAW,EAAE,oBAAoB;oBACjC,SAAS,EAAE,QAAQ;iBACpB;gBACD;oBACE,IAAI,EAAE,SAAS;oBACf,GAAG,EAAE,MAAM;oBACX,IAAI,EAAE,CAAC,qBAAqB,EAAE,QAAQ,EAAE,oBAAoB,EAAE,CAAC;oBAC/D,WAAW,EAAE,mBAAmB;oBAChC,SAAS,EAAE,QAAQ;iBACpB;aACF,CAAC;QACJ,CAAC;QAED,WAAW,EAAE,GAAG,EAAE,CAAC,EAAE;QAErB,OAAO,CAAC,OAAwB;YAC9B,OAAO,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QACvD,CAAC;QAED,gBAAgB;YACd,OAAO;gBACL,SAAS,EAAE,UAAU;gBACrB,iBAAiB,EAAE,mBAAmB;aACvC,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,MAAM,UAAU,GAAG,kBAAkB,EAAE,CAAC;IACxC,OAAO;QACL,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,eAAe,EAAE,UAAU,CAAC,eAAe;QAC3C,eAAe;YACb,OAAO,UAAU,CAAC,YAAY,CAAC;QACjC,CAAC;QACD,GAAG,EAAE,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC;QAClF,UAAU;KACX,CAAC;AACJ,CAAC"}

package/dist/lib/agents/fx/run.mjs

@@ -0,0 +1,150 @@
+/**
+ * fx in-sandbox runner. This file must remain zero-dependency.
+ */
+
+import { spawnSync } from 'node:child_process';
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+/** @param {{prompt:string}} input */
+export function buildFxCliArgs(input) {
+  return ['ask', '--yolo', '--json', '--no-color', '--', input.prompt];
+}
+
+/** @param {string} raw */
+export function parseFxAskResult(raw) {
+  if (!raw || !raw.trim()) return null;
+  try {
+    const value = JSON.parse(raw.trim());
+    if (
+      value &&
+      typeof value === 'object' &&
+      typeof value.output === 'string' &&
+      typeof value.exit_code === 'number' &&
+      Array.isArray(value.tool_calls)
+    ) {
+      return value;
+    }
+  } catch {
+    // The caller reports malformed stdout as an agent failure.
+  }
+  return null;
+}
+
+/** @param {string} raw */
+export function isFxSessionDetail(raw) {
+  if (!raw || !raw.trim()) return false;
+  try {
+    const value = JSON.parse(raw.trim());
+    return value?.kind === 'session_detail' && Array.isArray(value.history);
+  } catch {
+    return false;
+  }
+}
+
+/** @param {{model?:string}} input */
+export function buildFxEnvironment(input) {
+  const env = { ...process.env, FX_AUTO_UPGRADE: '0' };
+  if (input.model) env.FX_MODEL = input.model;
+  else delete env.FX_MODEL;
+  return env;
+}
+
+/**
+ * @param {import('../plugin/contract.js').AgentRunInput} input
+ * @returns {Promise<import('../plugin/contract.js').RunnerResult>}
+ */
+export async function runAgent(input) {
+  const binary = join(input.cwd, '__agent_eval__', 'bin', 'fx');
+  const env = buildFxEnvironment(input);
+  const res = spawnSync(binary, buildFxCliArgs(input), {
+    cwd: input.cwd,
+    env,
+    encoding: 'utf8',
+    maxBuffer: 64 * 1024 * 1024,
+  });
+
+  const stdout = res.stdout || '';
+  const stderr = res.stderr || '';
+  const output = stdout + stderr;
+  const askResult = parseFxAskResult(stdout);
+  const processExitCode = res.status == null ? -1 : res.status;
+  const agentExitCode = askResult?.exit_code ?? processExitCode;
+
+  // The supported session projection is richer than fx ask's final summary.
+  // Fall back to the ask JSON when a session cannot be read.
+  let transcript = askResult ? stdout.trim() : null;
+  if (askResult?.session_id) {
+    const session = spawnSync(binary, ['session', '--id', askResult.session_id, '--json'], {
+      cwd: input.cwd,
+      env,
+      encoding: 'utf8',
+      maxBuffer: 64 * 1024 * 1024,
+    });
+    if (session.status === 0 && isFxSessionDetail(session.stdout || '')) {
+      transcript = session.stdout.trim();
+    }
+  }
+
+  const ok = !res.error && processExitCode === 0 && askResult?.exit_code === 0;
+  if (!ok) {
+    const errorLines = output.trim().split('\n').slice(-5).join('\n');
+    const error = askResult?.error || (res.error ? `Failed to run fx: ${res.error.message}` : null);
+    return {
+      ok: false,
+      output,
+      transcript,
+      observedModel: askResult?.model || null,
+      error: error || errorLines || `fx exited with code ${agentExitCode}`,
+      agentExitCode,
+    };
+  }
+
+  return {
+    ok: true,
+    output,
+    transcript,
+    observedModel: askResult.model || null,
+    error: null,
+    agentExitCode,
+  };
+}
+
+const isMain = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
+
+if (isMain) {
+  const input = JSON.parse(process.argv[2]);
+  let result;
+  try {
+    result = await runAgent(input);
+  } catch (error) {
+    result = {
+      ok: false,
+      output: '',
+      transcript: null,
+      observedModel: null,
+      error: error && error.message ? error.message : String(error),
+      agentExitCode: -1,
+    };
+  }
+
+  try {
+    mkdirSync(dirname(input.resultPath), { recursive: true });
+    writeFileSync(input.resultPath, JSON.stringify(result));
+  } catch {
+    // The marker below remains available when the result file cannot be written.
+  }
+
+  process.stdout.write(
+    '__AGENT_RESULT__ ' +
+      JSON.stringify({
+        ok: result.ok,
+        observedModel: result.observedModel,
+        error: result.error,
+        agentExitCode: result.agentExitCode,
+      }) +
+      '\n'
+  );
+  process.exit(0);
+}

package/dist/lib/agents/index.d.ts

@@ -3,8 +3,11 @@
  */
 import { registerAgent, getAgent, listAgents, hasAgent } from './registry.js';
 import type { AgentType } from '../types.js';
-/** Validate bundled-skill isolation for every agent used by a run. */
-export declare function assertRunBundledSkillsControl(agentName: AgentType, requested?: boolean, judgeAgentName?: AgentType): void;
+/** Validate opt-in runtime controls for every agent used by a run. */
+export declare function assertRunRuntimeControls(agentName: AgentType, controls: {
+    disableBundledSkills?: boolean;
+    webResearch?: boolean;
+}, judgeAgentName?: AgentType): void;
 export { registerAgent, getAgent, listAgents, hasAgent };
 export type { Agent, AgentRunOptions, AgentRunResult } from './types.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/lib/agents/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAO9E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAW7C,sEAAsE;AACtE,wBAAgB,6BAA6B,CAC3C,SAAS,EAAE,SAAS,EACpB,SAAS,CAAC,EAAE,OAAO,EACnB,cAAc,CAAC,EAAE,SAAS,GACzB,IAAI,CAKN;AAGD,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAGzD,YAAY,EAAE,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAY9E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAY7C,sEAAsE;AACtE,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,SAAS,EACpB,QAAQ,EAAE;IAAE,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAAC,WAAW,CAAC,EAAE,OAAO,CAAA;CAAE,EACnE,cAAc,CAAC,EAAE,SAAS,GACzB,IAAI,CAUN;AAGD,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAGzD,YAAY,EAAE,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}

package/dist/lib/agents/index.js

@@ -5,22 +5,28 @@ import { registerAgent, getAgent, listAgents, hasAgent } from './registry.js';
 import { createClaudeCodeAgent } from './claude-code/agent.js';
 import { createCodexAgent } from './codex/agent.js';
 import { createOpenCodeAgent } from './opencode/agent.js';
+import { createFxAgent } from './fx/agent.js';
 import { createGeminiAgent } from './gemini/agent.js';
 import { createCursorAgent } from './cursor/agent.js';
-import { assertBundledSkillsControl } from './plugin/contract.js';
+import { assertBundledSkillsControl, assertCrossAgentJudgeSupport, assertWebResearchControl, } from './plugin/contract.js';
 // Register all agent variants (Vercel AI Gateway + Direct API)
 registerAgent(createClaudeCodeAgent({ useVercelAiGateway: true })); // vercel-ai-gateway/claude-code
 registerAgent(createClaudeCodeAgent({ useVercelAiGateway: false })); // claude-code
 registerAgent(createCodexAgent({ useVercelAiGateway: true })); // vercel-ai-gateway/codex
 registerAgent(createCodexAgent({ useVercelAiGateway: false })); // codex
 registerAgent(createOpenCodeAgent()); // vercel-ai-gateway/opencode
+registerAgent(createFxAgent()); // vercel-ai-gateway/fx
 registerAgent(createGeminiAgent()); // gemini
 registerAgent(createCursorAgent()); // cursor
-/** Validate bundled-skill isolation for every agent used by a run. */
-export function assertRunBundledSkillsControl(agentName, requested, judgeAgentName) {
-    const agentNames = new Set([agentName, judgeAgentName ?? agentName]);
-    for (const name of agentNames) {
-        assertBundledSkillsControl(getAgent(name).definition, requested);
+/** Validate opt-in runtime controls for every agent used by a run. */
+export function assertRunRuntimeControls(agentName, controls, judgeAgentName) {
+    const definition = getAgent(agentName).definition;
+    assertBundledSkillsControl(definition, controls.disableBundledSkills);
+    assertWebResearchControl(definition, controls.webResearch);
+    if (judgeAgentName && judgeAgentName !== agentName) {
+        const judgeDefinition = getAgent(judgeAgentName).definition;
+        assertBundledSkillsControl(judgeDefinition, controls.disableBundledSkills);
+        assertCrossAgentJudgeSupport(judgeDefinition);
     }
 }
 // Re-export registry functions

package/dist/lib/agents/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/lib/agents/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAGlE,+DAA+D;AAC/D,aAAa,CAAC,qBAAqB,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAG,gCAAgC;AACtG,aAAa,CAAC,qBAAqB,CAAC,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAE,cAAc;AACpF,aAAa,CAAC,gBAAgB,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAQ,0BAA0B;AAChG,aAAa,CAAC,gBAAgB,CAAC,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAO,QAAQ;AAC9E,aAAa,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAiC,6BAA6B;AACnG,aAAa,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAmC,SAAS;AAC/E,aAAa,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAmC,SAAS;AAE/E,sEAAsE;AACtE,MAAM,UAAU,6BAA6B,CAC3C,SAAoB,EACpB,SAAmB,EACnB,cAA0B;IAE1B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,cAAc,IAAI,SAAS,CAAC,CAAC,CAAC;IACrE,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,0BAA0B,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACnE,CAAC;AACH,CAAC;AAED,+BAA+B;AAC/B,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/lib/agents/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EACL,0BAA0B,EAC1B,4BAA4B,EAC5B,wBAAwB,GACzB,MAAM,sBAAsB,CAAC;AAG9B,+DAA+D;AAC/D,aAAa,CAAC,qBAAqB,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAG,gCAAgC;AACtG,aAAa,CAAC,qBAAqB,CAAC,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAE,cAAc;AACpF,aAAa,CAAC,gBAAgB,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAQ,0BAA0B;AAChG,aAAa,CAAC,gBAAgB,CAAC,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAO,QAAQ;AAC9E,aAAa,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAiC,6BAA6B;AACnG,aAAa,CAAC,aAAa,EAAE,CAAC,CAAC,CAAuC,uBAAuB;AAC7F,aAAa,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAmC,SAAS;AAC/E,aAAa,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAmC,SAAS;AAE/E,sEAAsE;AACtE,MAAM,UAAU,wBAAwB,CACtC,SAAoB,EACpB,QAAmE,EACnE,cAA0B;IAE1B,MAAM,UAAU,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC;IAClD,0BAA0B,CAAC,UAAU,EAAE,QAAQ,CAAC,oBAAoB,CAAC,CAAC;IACtE,wBAAwB,CAAC,UAAU,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;IAE3D,IAAI,cAAc,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;QACnD,MAAM,eAAe,GAAG,QAAQ,CAAC,cAAc,CAAC,CAAC,UAAU,CAAC;QAC5D,0BAA0B,CAAC,eAAe,EAAE,QAAQ,CAAC,oBAAoB,CAAC,CAAC;QAC3E,4BAA4B,CAAC,eAAe,CAAC,CAAC;IAChD,CAAC;AACH,CAAC;AAED,+BAA+B;AAC/B,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC"}

package/dist/lib/agents/plugin/contract.d.ts

@@ -78,6 +78,10 @@ export interface AgentDefinition {
     /** Whether the CLI can disable vendor-bundled skills, or has none to disable.
      * Omitted means the opt-in control is unsupported. */
     bundledSkillsControl?: 'configurable' | 'not-applicable';
+    /** Whether this adapter currently supports only research-enabled runs. */
+    requiresWebResearch?: boolean;
+    /** Whether another agent may select this adapter as its pinned judge. */
+    supportsCrossAgentJudge?: boolean;
     /**
      * Which host env var the apiKey is read from. Mirrors the old getApiKeyEnvVar()
      * EXACTLY, including claude-code's 3-way runtime branch (gateway →
@@ -115,6 +119,10 @@ export interface AgentDefinition {
 }
 /** Reject an opt-in capability when an agent cannot honor it. */
 export declare function assertBundledSkillsControl(definition: AgentDefinition, requested?: boolean): void;
+/** Reject a run when an adapter requires the research tool surface. */
+export declare function assertWebResearchControl(definition: AgentDefinition, requested?: boolean): void;
+/** Reject an adapter that cannot serve as another agent's pinned judge. */
+export declare function assertCrossAgentJudgeSupport(definition: AgentDefinition): void;
 /**
  * What the host passes INTO run.mjs (serialized as argv[2] JSON).
  *

package/dist/lib/agents/plugin/contract.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"contract.d.ts","sourceRoot":"","sources":["../../../../src/lib/agents/plugin/contract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC1E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD;;;;;;;GAOG;AACH,MAAM,WAAW,WAAW;IAC1B,qFAAqF;IACrF,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC;IAC1B,0BAA0B;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,wBAAwB;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,sFAAsF;IACtF,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,SAAS,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAChC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,kFAAkF;IAClF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+FAA+F;IAC/F,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,sFAAsF;IACtF,IAAI,EAAE,MAAM,CAAC;IACb,qEAAqE;IACrE,WAAW,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,YAAY,EAAE,SAAS,CAAC;IACxB,uFAAuF;IACvF,aAAa,EAAE,MAAM,CAAC;IACtB,8FAA8F;IAC9F,UAAU,EAAE,MAAM,CAAC;IACnB;0DACsD;IACtD,oBAAoB,CAAC,EAAE,cAAc,GAAG,gBAAgB,CAAC;IAEzD;;;;;OAKG;IACH,eAAe,IAAI,MAAM,CAAC;IAE1B,qFAAqF;IACrF,OAAO,CAAC,OAAO,EAAE,eAAe,GAAG,WAAW,EAAE,CAAC;IAEjD,uFAAuF;IACvF,WAAW,CAAC,OAAO,EAAE,eAAe,GAAG,UAAU,EAAE,CAAC;IAEpD;;;;OAIG;IACH,OAAO,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE1D;;;;;;;;;OASG;IACH,WAAW,CAAC,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhE;;;;OAIG;IACH,gBAAgB,CAAC,CAAC,MAAM,EAAE,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;CAC1F;AAED,iEAAiE;AACjE,wBAAgB,0BAA0B,CACxC,UAAU,EAAE,eAAe,EAC3B,SAAS,CAAC,EAAE,OAAO,GAClB,IAAI,CAIN;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,2FAA2F;IAC3F,MAAM,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oFAAoF;IACpF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6EAA6E;IAC7E,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,6CAA6C;IAC7C,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,qFAAqF;IACrF,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,yFAAyF;IACzF,GAAG,EAAE,MAAM,CAAC;IACZ,uEAAuE;IACvE,UAAU,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B,uCAAuC;IACvC,EAAE,EAAE,OAAO,CAAC;IACZ,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,+EAA+E;IAC/E,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,kFAAkF;IAClF,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,8EAA8E;IAC9E,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;;;;OAQG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}
+{"version":3,"file":"contract.d.ts","sourceRoot":"","sources":["../../../../src/lib/agents/plugin/contract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC1E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD;;;;;;;GAOG;AACH,MAAM,WAAW,WAAW;IAC1B,qFAAqF;IACrF,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC;IAC1B,0BAA0B;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,wBAAwB;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,sFAAsF;IACtF,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,SAAS,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAChC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,kFAAkF;IAClF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+FAA+F;IAC/F,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,sFAAsF;IACtF,IAAI,EAAE,MAAM,CAAC;IACb,qEAAqE;IACrE,WAAW,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,YAAY,EAAE,SAAS,CAAC;IACxB,uFAAuF;IACvF,aAAa,EAAE,MAAM,CAAC;IACtB,8FAA8F;IAC9F,UAAU,EAAE,MAAM,CAAC;IACnB;0DACsD;IACtD,oBAAoB,CAAC,EAAE,cAAc,GAAG,gBAAgB,CAAC;IACzD,0EAA0E;IAC1E,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,yEAAyE;IACzE,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAElC;;;;;OAKG;IACH,eAAe,IAAI,MAAM,CAAC;IAE1B,qFAAqF;IACrF,OAAO,CAAC,OAAO,EAAE,eAAe,GAAG,WAAW,EAAE,CAAC;IAEjD,uFAAuF;IACvF,WAAW,CAAC,OAAO,EAAE,eAAe,GAAG,UAAU,EAAE,CAAC;IAEpD;;;;OAIG;IACH,OAAO,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE1D;;;;;;;;;OASG;IACH,WAAW,CAAC,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhE;;;;OAIG;IACH,gBAAgB,CAAC,CAAC,MAAM,EAAE,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;CAC1F;AAED,iEAAiE;AACjE,wBAAgB,0BAA0B,CACxC,UAAU,EAAE,eAAe,EAC3B,SAAS,CAAC,EAAE,OAAO,GAClB,IAAI,CAIN;AAED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,UAAU,EAAE,eAAe,EAC3B,SAAS,CAAC,EAAE,OAAO,GAClB,IAAI,CAIN;AAED,2EAA2E;AAC3E,wBAAgB,4BAA4B,CAAC,UAAU,EAAE,eAAe,GAAG,IAAI,CAI9E;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,2FAA2F;IAC3F,MAAM,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oFAAoF;IACpF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6EAA6E;IAC7E,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,6CAA6C;IAC7C,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,qFAAqF;IACrF,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,yFAAyF;IACzF,GAAG,EAAE,MAAM,CAAC;IACZ,uEAAuE;IACvE,UAAU,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B,uCAAuC;IACvC,EAAE,EAAE,OAAO,CAAC;IACZ,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,+EAA+E;IAC/E,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,kFAAkF;IAClF,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,8EAA8E;IAC9E,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;;;;OAQG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}

package/dist/lib/agents/plugin/contract.js

@@ -24,4 +24,16 @@ export function assertBundledSkillsControl(definition, requested) {
         throw new Error(`Agent ${definition.name} does not support disableBundledSkills`);
     }
 }
+/** Reject a run when an adapter requires the research tool surface. */
+export function assertWebResearchControl(definition, requested) {
+    if (definition.requiresWebResearch && !requested) {
+        throw new Error(`Agent ${definition.name} requires webResearch: true`);
+    }
+}
+/** Reject an adapter that cannot serve as another agent's pinned judge. */
+export function assertCrossAgentJudgeSupport(definition) {
+    if (definition.supportsCrossAgentJudge === false) {
+        throw new Error(`Agent ${definition.name} does not support cross-agent judging`);
+    }
+}
 //# sourceMappingURL=contract.js.map

package/dist/lib/agents/plugin/contract.js.map

@@ -1 +1 @@
-{"version":3,"file":"contract.js","sourceRoot":"","sources":["../../../../src/lib/agents/plugin/contract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AA2GH,iEAAiE;AACjE,MAAM,UAAU,0BAA0B,CACxC,UAA2B,EAC3B,SAAmB;IAEnB,IAAI,SAAS,IAAI,CAAC,UAAU,CAAC,oBAAoB,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,SAAS,UAAU,CAAC,IAAI,wCAAwC,CAAC,CAAC;IACpF,CAAC;AACH,CAAC"}
+{"version":3,"file":"contract.js","sourceRoot":"","sources":["../../../../src/lib/agents/plugin/contract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AA+GH,iEAAiE;AACjE,MAAM,UAAU,0BAA0B,CACxC,UAA2B,EAC3B,SAAmB;IAEnB,IAAI,SAAS,IAAI,CAAC,UAAU,CAAC,oBAAoB,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,SAAS,UAAU,CAAC,IAAI,wCAAwC,CAAC,CAAC;IACpF,CAAC;AACH,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,wBAAwB,CACtC,UAA2B,EAC3B,SAAmB;IAEnB,IAAI,UAAU,CAAC,mBAAmB,IAAI,CAAC,SAAS,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,SAAS,UAAU,CAAC,IAAI,6BAA6B,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,4BAA4B,CAAC,UAA2B;IACtE,IAAI,UAAU,CAAC,uBAAuB,KAAK,KAAK,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,SAAS,UAAU,CAAC,IAAI,uCAAuC,CAAC,CAAC;IACnF,CAAC;AACH,CAAC"}

package/dist/lib/agents/plugin/orchestrator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../../../src/lib/agents/plugin/orchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAwBnE,OAAO,EAEL,KAAK,eAAe,EAGrB,MAAM,eAAe,CAAC;AAoBvB,+EAA+E;AAC/E,UAAU,kBAAkB;IAC1B,oFAAoF;IACpF,UAAU,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,2EAA2E;IAC3E,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,uEAAuE;IACvE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CACvC;AAED,UAAU,YAAY;IACpB,iFAAiF;IACjF,QAAQ,EAAE,eAAe,CAAC;IAC1B,sFAAsF;IACtF,YAAY,EAAE,eAAe,CAAC;IAC9B,qFAAqF;IACrF,MAAM,EAAE,OAAO,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,gFAAgF;IAChF,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,2DAA2D;IAC3D,MAAM,EAAE,kBAAkB,CAAC;CAC5B;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,GAAG,YAAY,CAyChG;AA0FD;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,eAAe,EACpB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,cAAc,CAAC,CAgBzB"}
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../../../src/lib/agents/plugin/orchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAwBnE,OAAO,EAIL,KAAK,eAAe,EAGrB,MAAM,eAAe,CAAC;AAoBvB,+EAA+E;AAC/E,UAAU,kBAAkB;IAC1B,oFAAoF;IACpF,UAAU,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,2EAA2E;IAC3E,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,uEAAuE;IACvE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CACvC;AAED,UAAU,YAAY;IACpB,iFAAiF;IACjF,QAAQ,EAAE,eAAe,CAAC;IAC1B,sFAAsF;IACtF,YAAY,EAAE,eAAe,CAAC;IAC9B,qFAAqF;IACrF,MAAM,EAAE,OAAO,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,gFAAgF;IAChF,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,2DAA2D;IAC3D,MAAM,EAAE,kBAAkB,CAAC;CAC5B;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,GAAG,YAAY,CA2ChG;AA0FD;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,eAAe,EACpB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,cAAc,CAAC,CAkBzB"}

package/dist/lib/agents/plugin/orchestrator.js

@@ -18,7 +18,7 @@ import { createSandbox, collectLocalFiles, splitTestFiles, verifyNoTestFiles, }
 import { runValidation, captureGeneratedFiles, createVitestConfig, initGitAndCommit, injectTranscriptContext, prepareNeutralWorkspace, resolveAgentApiKey, EVAL_HELPER_PATH, JUDGE_TRANSCRIPT_FILE, JUDGE_CONFIG_PATH, JUDGE_RUNNER_PATH, } from '../shared.js';
 import { redactRunResult } from '../redact.js';
 import { getAgent } from '../registry.js';
-import { assertBundledSkillsControl, } from './contract.js';
+import { assertBundledSkillsControl, assertCrossAgentJudgeSupport, assertWebResearchControl, } from './contract.js';
 /** Well-known paths inside the sandbox for the runner + its result file. */
 const RUNNER_PATH = '__agent_eval__/run.mjs';
 const RESULT_PATH = '__agent_eval__/agent-result.json';
@@ -44,6 +44,7 @@ const EVAL_HELPER_DISK_PATH = fileURLToPath(new URL('../eval-helper.mjs', import
 export function resolveJudgeRuntime(def, options) {
     const spec = options.judge;
     assertBundledSkillsControl(def, options.disableBundledSkills);
+    assertWebResearchControl(def, options.webResearch);
     // Same harness as codegen (default, or judge.agent omitted/equal): reuse run.mjs,
     // just pin the model when asked. Identical to pre-feature behavior when unset.
     if (!spec || (spec.agent ?? def.name) === def.name) {
@@ -65,6 +66,7 @@ export function resolveJudgeRuntime(def, options) {
     // Pinned to a DIFFERENT agent — resolve its definition + key + runner.
     const judgeDef = getAgent(spec.agent).definition;
     assertBundledSkillsControl(judgeDef, options.disableBundledSkills);
+    assertCrossAgentJudgeSupport(judgeDef);
     const judgeApiKey = resolveAgentApiKey(judgeDef.getApiKeyEnvVar) ?? '';
     const judgeOptions = { ...options, model: spec.model, apiKey: judgeApiKey };
     return {
@@ -172,6 +174,8 @@ async function readRunnerResult(sandbox, resultPath, nodeResult) {
  * transcript.
  */
 export async function runWithDefinition(def, fixturePath, options) {
+    assertBundledSkillsControl(def, options.disableBundledSkills);
+    assertWebResearchControl(def, options.webResearch);
     const result = await runOnce(def, fixturePath, options);
     // A judge pinned to a different agent authenticates with its own key, and it can
     // reach the transcript too, so redact both. Resolving the judge can throw (an