@agentv/core 4.25.1 → 4.25.3-next.1

package/dist/{chunk-IXTJEXWN.js → chunk-EVEZQXIS.js}

@@ -17,14 +17,19 @@ import {
   readTextFile,
   resolveDelegatedTargetDefinition,
   resolveTargetDefinition
-} from "./chunk-6HLBKYE2.js";
+} from "./chunk-CALQDF2Y.js";
 import {
   execFileWithStdin,
   execShellWithStdin
 } from "./chunk-3WGHC7LC.js";
 import {
-  AgentvProvider
-} from "./chunk-PRNXHNLF.js";
+  AgentvProvider,
+  AnthropicProvider,
+  AzureProvider,
+  GeminiProvider,
+  OpenAIProvider,
+  OpenRouterProvider
+} from "./chunk-5XV3FAAD.js";
 
 // src/evaluation/loaders/ts-eval-loader.ts
 import path46 from "node:path";
@@ -730,6 +735,8 @@ var CodeGrader = class {
     const env = proxyEnv || workspaceEnv ? { ...proxyEnv, ...workspaceEnv } : void 0;
     try {
       let stdout;
+      let exitCode = 0;
+      let execStderr = "";
       if (context.dockerConfig) {
         const { DockerWorkspaceProvider } = await import("./docker-workspace-RPPXBT27.js");
         const dockerProvider = new DockerWorkspaceProvider(context.dockerConfig);
@@ -738,31 +745,42 @@ var CodeGrader = class {
           stdin: inputPayload,
           repoCheckouts: getRepoCheckoutTargets(context.evalCase.workspace?.repos)
         });
-        if (result.exitCode !== 0) {
-          const trimmedErr = result.stderr.trim();
-          throw new Error(
-            trimmedErr.length > 0 ? `Code evaluator exited with code ${result.exitCode}: ${trimmedErr}` : `Code evaluator exited with code ${result.exitCode}`
-          );
-        }
+        exitCode = result.exitCode;
         stdout = result.stdout.trim();
+        execStderr = result.stderr;
       } else {
-        stdout = await executeScript(
+        const result = await runScriptRaw(
           this.command,
           inputPayload,
           this.agentTimeoutMs,
           this.cwd,
           env
         );
+        exitCode = result.exitCode;
+        stdout = result.stdout.trim();
+        execStderr = result.stderr;
       }
-      const parsed = parseJsonSafe(stdout);
-      const score = clampScore(typeof parsed?.score === "number" ? parsed.score : 0);
-      const assertions = Array.isArray(parsed?.assertions) ? parsed.assertions.filter(
+      const looksLikeJson = stdout.startsWith("{") || stdout.startsWith("[");
+      const hasStderr = execStderr.trim().length > 0;
+      if (exitCode !== 0 && (looksLikeJson || hasStderr)) {
+        const trimmedErr = formatStderr(execStderr);
+        throw new Error(
+          trimmedErr.length > 0 ? `Code evaluator exited with code ${exitCode}: ${trimmedErr}` : `Code evaluator exited with code ${exitCode}`
+        );
+      }
+      const rawParsed = parseJsonSafe(stdout);
+      const parsed = rawParsed != null && typeof rawParsed === "object" && !Array.isArray(rawParsed) ? rawParsed : void 0;
+      const passed = exitCode === 0;
+      const assertions = parsed != null && Array.isArray(parsed?.assertions) ? parsed.assertions.filter(
         (a) => typeof a === "object" && a !== null && typeof a.text === "string"
       ).map((a) => ({
         text: String(a.text),
         passed: Boolean(a.passed),
         ...typeof a.evidence === "string" ? { evidence: a.evidence } : {}
-      })) : [];
+      })) : parsed == null ? [{ text: stdout.trim() || (passed ? "exit 0" : `exit ${exitCode}`), passed }] : [];
+      const score = parsed != null ? clampScore(
+        typeof parsed.score === "number" ? parsed.score : assertions.length > 0 ? assertions.filter((a) => a.passed).length / assertions.length : 0
+      ) : passed ? 1 : 0;
       const details = parsed?.details && typeof parsed.details === "object" && !Array.isArray(parsed.details) ? parsed.details : void 0;
       const proxyUsage = getProxyUsage?.();
       const graderRawRequest = {
@@ -820,8 +838,17 @@ var CodeGrader = class {
     }
   }
 };
+async function runScriptRaw(scriptPath, input, agentTimeoutMs, cwd, env) {
+  return typeof scriptPath === "string" ? execShellWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs, env }) : execFileWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs, env });
+}
 async function executeScript(scriptPath, input, agentTimeoutMs, cwd, env) {
-  const { stdout, stderr, exitCode } = typeof scriptPath === "string" ? await execShellWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs, env }) : await execFileWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs, env });
+  const { stdout, stderr, exitCode } = await runScriptRaw(
+    scriptPath,
+    input,
+    agentTimeoutMs,
+    cwd,
+    env
+  );
   if (exitCode !== 0) {
     const trimmedErr = formatStderr(stderr);
     throw new Error(
@@ -841,13 +868,9 @@ function formatStderr(stderr) {
 ${tail}`;
 }
 
-// src/evaluation/graders/composite.ts
-import { generateText as generateText2 } from "ai";
-
 // src/evaluation/graders/llm-grader.ts
 import fs from "node:fs/promises";
 import path3 from "node:path";
-import { generateText, stepCountIs, tool } from "ai";
 import { z } from "zod";
 
 // src/evaluation/content-preprocessor.ts
@@ -1357,18 +1380,15 @@ ${context.toolCalls}`;
     }
   }
   // ---------------------------------------------------------------------------
-  // Built-in agent mode (agentv provider — AI SDK generateText with filesystem tools)
+  // Built-in agent mode (agentv provider — provider.invoke() with filesystem tools)
   // ---------------------------------------------------------------------------
   /**
-   * Built-in mode: Uses Vercel AI SDK generateText() with sandboxed filesystem tools.
+   * Built-in mode: drives the grader through provider.invoke() with the
+   * sandboxed filesystem tools and a step budget. The pi-ai-backed agentv
+   * provider runs the agent loop (tool call → tool execute → next model
+   * turn) until the model stops requesting tools or maxSteps is hit.
    */
   async evaluateBuiltIn(context, graderProvider) {
-    const model = graderProvider.asLanguageModel?.();
-    if (!model) {
-      throw new Error(
-        `Grader provider '${graderProvider.targetName}' does not support asLanguageModel() \u2014 required for built-in agent mode`
-      );
-    }
     const workspacePath = context.workspacePath;
     if (!workspacePath) {
       throw new Error(
@@ -1387,18 +1407,21 @@ ${context.toolCalls}`;
       maxSteps: this.maxSteps
     };
     try {
-      const { text, steps } = await generateText({
-        model,
-        system: systemPrompt,
-        prompt: userPrompt,
+      const response = await graderProvider.invoke({
+        question: userPrompt,
+        systemPrompt,
+        evalCaseId: context.evalCase.id,
+        attempt: context.attempt,
+        temperature: this.temperature ?? 0,
         tools: fsTools,
-        stopWhen: stepCountIs(this.maxSteps),
-        temperature: this.temperature ?? 0
+        maxSteps: this.maxSteps
       });
-      const toolCallCount = steps.reduce((count, step) => count + (step.toolCalls?.length ?? 0), 0);
+      const text = extractLastAssistantContent(response.output);
+      const stepCount = response.steps?.count ?? 1;
+      const toolCallCount = response.steps?.toolCallCount ?? 0;
       const details = {
         mode: "built-in",
-        steps: steps.length,
+        steps: stepCount,
         tool_calls: toolCallCount
       };
       return this.parseAgentResult(
@@ -1850,43 +1873,14 @@ ${outputSchema}`;
   }
   async generateStructuredResponse(options) {
     const { context, graderProvider, systemPrompt, userPrompt, images } = options;
-    const model = graderProvider.asLanguageModel?.();
-    if (model) {
-      const modelOptions = {
-        ...this.maxOutputTokens ? { maxTokens: this.maxOutputTokens } : {},
-        ...typeof this.temperature === "number" ? { temperature: this.temperature } : {}
-      };
-      const hasImages = images && images.length > 0;
-      const result = hasImages ? await generateText({
-        model,
-        system: systemPrompt,
-        messages: [
-          {
-            role: "user",
-            content: [
-              { type: "text", text: userPrompt },
-              ...toAiSdkImageParts(images)
-            ]
-          }
-        ],
-        ...modelOptions
-      }) : await generateText({
-        model,
-        system: systemPrompt,
-        prompt: userPrompt,
-        ...modelOptions
-      });
-      const rawUsage = result.usage;
-      const tokenUsage = rawUsage?.inputTokens != null && rawUsage?.outputTokens != null ? { input: rawUsage.inputTokens, output: rawUsage.outputTokens } : void 0;
-      return { text: result.text, tokenUsage };
-    }
     const response = await graderProvider.invoke({
       question: userPrompt,
       systemPrompt,
       evalCaseId: context.evalCase.id,
       attempt: context.attempt,
       maxOutputTokens: this.maxOutputTokens,
-      temperature: this.temperature
+      temperature: this.temperature,
+      ...images && images.length > 0 ? { images } : {}
     });
     return {
       text: extractLastAssistantContent(response.output),
@@ -2083,13 +2077,6 @@ function extractImageBlocks(messages) {
   }
   return images;
 }
-function toAiSdkImageParts(images) {
-  return images.map((img) => ({
-    type: "image",
-    image: img.source,
-    mediaType: img.media_type || void 0
-  }));
-}
 function resolveSandboxed(basePath, relativePath) {
   const resolved = path3.resolve(basePath, relativePath);
   if (!resolved.startsWith(basePath + path3.sep) && resolved !== basePath) {
@@ -2098,15 +2085,24 @@ function resolveSandboxed(basePath, relativePath) {
   return resolved;
 }
 function createFilesystemTools(workspacePath) {
-  return {
-    list_files: tool({
+  return [
+    {
+      name: "list_files",
       description: "List files and directories at a relative path within the workspace. Returns names only (single level, no recursion).",
-      inputSchema: z.object({
-        path: z.string().describe('Relative path within workspace (use "." for root)').default(".")
-      }),
+      parameters: {
+        type: "object",
+        properties: {
+          path: {
+            type: "string",
+            description: 'Relative path within workspace (use "." for root)',
+            default: "."
+          }
+        }
+      },
       execute: async (input) => {
+        const args = input ?? {};
         try {
-          const resolved = resolveSandboxed(workspacePath, input.path);
+          const resolved = resolveSandboxed(workspacePath, args.path ?? ".");
           const entries = await fs.readdir(resolved, { withFileTypes: true });
           return entries.map((e) => ({
             name: e.name,
@@ -2116,18 +2112,25 @@ function createFilesystemTools(workspacePath) {
           return { error: error instanceof Error ? error.message : String(error) };
         }
       }
-    }),
-    read_file: tool({
+    },
+    {
+      name: "read_file",
       description: "Read the content of a file at a relative path within the workspace. Large files are truncated at 50KB.",
-      inputSchema: z.object({
-        path: z.string().describe("Relative path to file within workspace")
-      }),
+      parameters: {
+        type: "object",
+        properties: {
+          path: { type: "string", description: "Relative path to file within workspace" }
+        },
+        required: ["path"]
+      },
       execute: async (input) => {
+        const args = input ?? {};
+        const relPath = args.path ?? "";
         try {
-          const resolved = resolveSandboxed(workspacePath, input.path);
+          const resolved = resolveSandboxed(workspacePath, relPath);
           const stat10 = await fs.stat(resolved);
           if (stat10.isDirectory()) {
-            return { error: `'${input.path}' is a directory, not a file` };
+            return { error: `'${relPath}' is a directory, not a file` };
           }
           const buffer = Buffer.alloc(Math.min(stat10.size, MAX_FILE_SIZE));
           const fd = await fs.open(resolved, "r");
@@ -2143,19 +2146,29 @@ function createFilesystemTools(workspacePath) {
           return { error: error instanceof Error ? error.message : String(error) };
         }
       }
-    }),
-    search_files: tool({
+    },
+    {
+      name: "search_files",
       description: "Search for a regex pattern across files in the workspace. Returns up to 20 matches. Skips binary files and node_modules/.git.",
-      inputSchema: z.object({
-        pattern: z.string().describe("Regex pattern to search for"),
-        path: z.string().describe('Relative path to search within (use "." for root)').default(".")
-      }),
+      parameters: {
+        type: "object",
+        properties: {
+          pattern: { type: "string", description: "Regex pattern to search for" },
+          path: {
+            type: "string",
+            description: 'Relative path to search within (use "." for root)',
+            default: "."
+          }
+        },
+        required: ["pattern"]
+      },
       execute: async (input) => {
+        const args = input ?? {};
         try {
-          const resolved = resolveSandboxed(workspacePath, input.path);
+          const resolved = resolveSandboxed(workspacePath, args.path ?? ".");
           let regex;
           try {
-            regex = new RegExp(input.pattern, "gi");
+            regex = new RegExp(args.pattern ?? "", "gi");
           } catch (regexErr) {
             return {
               error: `Invalid regex pattern: ${regexErr instanceof Error ? regexErr.message : String(regexErr)}`
@@ -2168,8 +2181,8 @@ function createFilesystemTools(workspacePath) {
           return { error: error instanceof Error ? error.message : String(error) };
         }
       }
-    })
-  };
+    }
+  ];
 }
 async function searchDirectory(dirPath, workspacePath, regex, matches) {
   if (matches.length >= MAX_SEARCH_MATCHES) return;
@@ -2449,25 +2462,6 @@ var CompositeGrader = class {
       target: graderProvider.targetName
     };
     try {
-      const model = graderProvider.asLanguageModel?.();
-      if (model) {
-        const { text } = await generateText2({
-          model,
-          system: systemPrompt,
-          prompt: userPrompt
-        });
-        const data2 = freeformEvaluationSchema.parse(parseJsonFromText(text));
-        const score2 = clampScore(data2.score);
-        const assertions2 = Array.isArray(data2.assertions) ? data2.assertions.slice(0, 8) : [];
-        return {
-          score: score2,
-          verdict: scoreToVerdict(score2),
-          assertions: assertions2,
-          expectedAspectCount: Math.max(assertions2.length, 1),
-          graderRawRequest,
-          scores
-        };
-      }
       const response = await graderProvider.invoke({
         question: userPrompt,
         systemPrompt,
@@ -2625,7 +2619,7 @@ var DEFAULT_EXPLORATION_TOOLS = [
 function explorationRatio(summary, explorationTools = DEFAULT_EXPLORATION_TOOLS) {
   if (summary.eventCount === 0) return void 0;
   const explorationCalls = explorationTools.reduce(
-    (sum, tool2) => sum + (summary.toolCalls[tool2] ?? 0),
+    (sum, tool) => sum + (summary.toolCalls[tool] ?? 0),
     0
   );
   return explorationCalls / summary.eventCount;
@@ -4261,422 +4255,6 @@ function runEqualsAssertion(output, value) {
   };
 }
 
-// src/evaluation/providers/ai-sdk.ts
-import { createAnthropic } from "@ai-sdk/anthropic";
-import { createAzure } from "@ai-sdk/azure";
-import { createGoogleGenerativeAI } from "@ai-sdk/google";
-import { createOpenAI } from "@ai-sdk/openai";
-import { createOpenRouter } from "@openrouter/ai-sdk-provider";
-import { generateText as generateText3 } from "ai";
-var DEFAULT_SYSTEM_PROMPT = "You are a careful assistant. Follow all provided instructions and do not fabricate results.";
-var OpenAIProvider = class {
-  constructor(targetName, config) {
-    this.config = config;
-    this.id = `openai:${targetName}`;
-    this.targetName = targetName;
-    this.defaults = {
-      temperature: config.temperature,
-      maxOutputTokens: config.maxOutputTokens
-    };
-    this.retryConfig = config.retry;
-    const openai = createOpenAI({
-      apiKey: config.apiKey,
-      baseURL: config.baseURL
-    });
-    this.model = config.apiFormat === "responses" ? openai(config.model) : openai.chat(config.model);
-  }
-  id;
-  kind = "openai";
-  targetName;
-  model;
-  defaults;
-  retryConfig;
-  async invoke(request) {
-    return invokeModel({
-      model: this.model,
-      request,
-      defaults: this.defaults,
-      retryConfig: this.retryConfig
-    });
-  }
-  asLanguageModel() {
-    return this.model;
-  }
-};
-var AzureProvider = class {
-  constructor(targetName, config) {
-    this.config = config;
-    this.id = `azure:${targetName}`;
-    this.targetName = targetName;
-    this.defaults = {
-      temperature: config.temperature,
-      maxOutputTokens: config.maxOutputTokens
-    };
-    this.retryConfig = config.retry;
-    const azure = createAzure(buildAzureOptions(config));
-    this.model = config.apiFormat === "responses" ? azure(config.deploymentName) : azure.chat(config.deploymentName);
-  }
-  id;
-  kind = "azure";
-  targetName;
-  model;
-  defaults;
-  retryConfig;
-  async invoke(request) {
-    return invokeModel({
-      model: this.model,
-      request,
-      defaults: this.defaults,
-      retryConfig: this.retryConfig
-    });
-  }
-  asLanguageModel() {
-    return this.model;
-  }
-};
-var OpenRouterProvider = class {
-  constructor(targetName, config) {
-    this.config = config;
-    this.id = `openrouter:${targetName}`;
-    this.targetName = targetName;
-    this.defaults = {
-      temperature: config.temperature,
-      maxOutputTokens: config.maxOutputTokens
-    };
-    this.retryConfig = config.retry;
-    const openrouter = createOpenRouter({
-      apiKey: config.apiKey
-    });
-    this.model = openrouter(config.model);
-  }
-  id;
-  kind = "openrouter";
-  targetName;
-  model;
-  defaults;
-  retryConfig;
-  async invoke(request) {
-    return invokeModel({
-      model: this.model,
-      request,
-      defaults: this.defaults,
-      retryConfig: this.retryConfig
-    });
-  }
-  asLanguageModel() {
-    return this.model;
-  }
-};
-var AnthropicProvider = class {
-  constructor(targetName, config) {
-    this.config = config;
-    this.id = `anthropic:${targetName}`;
-    this.targetName = targetName;
-    this.defaults = {
-      temperature: config.temperature,
-      maxOutputTokens: config.maxOutputTokens,
-      thinkingBudget: config.thinkingBudget
-    };
-    this.retryConfig = config.retry;
-    const anthropic = createAnthropic({
-      apiKey: config.apiKey
-    });
-    this.model = anthropic(config.model);
-  }
-  id;
-  kind = "anthropic";
-  targetName;
-  model;
-  defaults;
-  retryConfig;
-  async invoke(request) {
-    const providerOptions = buildAnthropicProviderOptions(this.defaults);
-    return invokeModel({
-      model: this.model,
-      request,
-      defaults: this.defaults,
-      retryConfig: this.retryConfig,
-      providerOptions
-    });
-  }
-  asLanguageModel() {
-    return this.model;
-  }
-};
-var GeminiProvider = class {
-  constructor(targetName, config) {
-    this.config = config;
-    this.id = `gemini:${targetName}`;
-    this.targetName = targetName;
-    this.defaults = {
-      temperature: config.temperature,
-      maxOutputTokens: config.maxOutputTokens
-    };
-    this.retryConfig = config.retry;
-    const google = createGoogleGenerativeAI({
-      apiKey: config.apiKey
-    });
-    this.model = google(config.model);
-  }
-  id;
-  kind = "gemini";
-  targetName;
-  model;
-  defaults;
-  retryConfig;
-  async invoke(request) {
-    return invokeModel({
-      model: this.model,
-      request,
-      defaults: this.defaults,
-      retryConfig: this.retryConfig
-    });
-  }
-  asLanguageModel() {
-    return this.model;
-  }
-};
-function buildAzureOptions(config) {
-  const options = {
-    apiKey: config.apiKey,
-    apiVersion: config.version,
-    // Chat completions still use deployment-scoped Azure URLs for compatibility
-    // with existing deployments. Responses API should use the SDK's v1 path.
-    useDeploymentBasedUrls: config.apiFormat !== "responses"
-  };
-  const baseURL = normalizeAzureBaseUrl(config.resourceName);
-  if (baseURL) {
-    options.baseURL = baseURL;
-  } else {
-    options.resourceName = config.resourceName;
-  }
-  return options;
-}
-function normalizeAzureBaseUrl(resourceName) {
-  const trimmed = resourceName.trim();
-  if (!/^https?:\/\//i.test(trimmed)) {
-    return void 0;
-  }
-  const withoutSlash = trimmed.replace(/\/+$/, "");
-  const normalized = withoutSlash.endsWith("/openai") ? withoutSlash : `${withoutSlash}/openai`;
-  return normalized;
-}
-function buildAnthropicProviderOptions(defaults) {
-  if (defaults.thinkingBudget === void 0) {
-    return void 0;
-  }
-  return {
-    anthropic: {
-      thinking: {
-        type: "enabled",
-        budgetTokens: defaults.thinkingBudget
-      }
-    }
-  };
-}
-function buildChatPrompt(request) {
-  const provided = request.chatPrompt?.length ? request.chatPrompt : void 0;
-  if (provided) {
-    const hasSystemMessage = provided.some((message) => message.role === "system");
-    if (hasSystemMessage) {
-      return provided;
-    }
-    const systemContent2 = resolveSystemContent(request);
-    return [{ role: "system", content: systemContent2 }, ...provided];
-  }
-  const systemContent = resolveSystemContent(request);
-  const userContent = request.question.trim();
-  const prompt = [
-    { role: "system", content: systemContent },
-    { role: "user", content: userContent }
-  ];
-  return prompt;
-}
-function resolveSystemContent(request) {
-  const systemSegments = [];
-  if (request.systemPrompt && request.systemPrompt.trim().length > 0) {
-    systemSegments.push(request.systemPrompt.trim());
-  } else {
-    systemSegments.push(DEFAULT_SYSTEM_PROMPT);
-  }
-  return systemSegments.join("\n\n");
-}
-function toModelMessages(chatPrompt) {
-  return chatPrompt.map((message) => {
-    if (message.role === "tool" || message.role === "function") {
-      const prefix = message.name ? `@[${message.name}]: ` : "@[Tool]: ";
-      return {
-        role: "assistant",
-        content: `${prefix}${message.content}`
-      };
-    }
-    if (message.role === "assistant" || message.role === "system" || message.role === "user") {
-      return {
-        role: message.role,
-        content: message.content
-      };
-    }
-    return {
-      role: "user",
-      content: message.content
-    };
-  });
-}
-function resolveModelSettings(request, defaults) {
-  const temperature = request.temperature ?? defaults.temperature;
-  const maxOutputTokens = request.maxOutputTokens ?? defaults.maxOutputTokens;
-  return {
-    temperature,
-    maxOutputTokens
-  };
-}
-async function invokeModel(options) {
-  const { model, request, defaults, retryConfig, providerOptions } = options;
-  const chatPrompt = buildChatPrompt(request);
-  const { temperature, maxOutputTokens } = resolveModelSettings(request, defaults);
-  const startTime = (/* @__PURE__ */ new Date()).toISOString();
-  const startMs = Date.now();
-  const result = await withRetry(
-    () => generateText3({
-      model,
-      messages: toModelMessages(chatPrompt),
-      temperature,
-      maxOutputTokens,
-      maxRetries: 0,
-      abortSignal: request.signal,
-      ...providerOptions ? { providerOptions } : {}
-    }),
-    retryConfig,
-    request.signal
-  );
-  const endTime = (/* @__PURE__ */ new Date()).toISOString();
-  const durationMs = Date.now() - startMs;
-  return mapResponse(result, { durationMs, startTime, endTime });
-}
-function mapResponse(result, timing) {
-  const content = result.text ?? "";
-  const rawUsage = result.totalUsage ?? result.usage;
-  const reasoning = rawUsage?.outputTokenDetails?.reasoningTokens ?? void 0;
-  const cached = rawUsage?.inputTokenDetails?.cacheReadTokens ?? void 0;
-  const tokenUsage = rawUsage?.inputTokens != null && rawUsage?.outputTokens != null ? {
-    input: rawUsage.inputTokens,
-    output: rawUsage.outputTokens,
-    ...reasoning != null ? { reasoning } : {},
-    ...cached != null ? { cached } : {}
-  } : void 0;
-  return {
-    raw: result,
-    usage: toJsonObject(rawUsage),
-    output: [{ role: "assistant", content }],
-    tokenUsage,
-    durationMs: timing?.durationMs,
-    startTime: timing?.startTime,
-    endTime: timing?.endTime
-  };
-}
-function toJsonObject(value) {
-  if (!value || typeof value !== "object") {
-    return void 0;
-  }
-  try {
-    return JSON.parse(JSON.stringify(value));
-  } catch {
-    return void 0;
-  }
-}
-function extractStatus(error) {
-  if (!error || typeof error !== "object") {
-    return void 0;
-  }
-  const candidate = error;
-  const directStatus = candidate.status ?? candidate.statusCode;
-  if (typeof directStatus === "number" && Number.isFinite(directStatus)) {
-    return directStatus;
-  }
-  const responseStatus = typeof candidate.response === "object" && candidate.response ? candidate.response.status : void 0;
-  if (typeof responseStatus === "number" && Number.isFinite(responseStatus)) {
-    return responseStatus;
-  }
-  const message = typeof candidate.message === "string" ? candidate.message : void 0;
-  if (message) {
-    const match = message.match(/HTTP\s+(\d{3})/i);
-    if (match) {
-      const parsed = Number.parseInt(match[1], 10);
-      if (Number.isFinite(parsed)) {
-        return parsed;
-      }
-    }
-  }
-  return void 0;
-}
-function isNetworkError(error) {
-  if (!error || typeof error !== "object") {
-    return false;
-  }
-  const candidate = error;
-  if (candidate.name === "AbortError") {
-    return false;
-  }
-  const code = candidate.code;
-  if (typeof code === "string" && /^E(AI|CONN|HOST|NET|PIPE|TIME|REFUSED|RESET)/i.test(code)) {
-    return true;
-  }
-  const message = typeof candidate.message === "string" ? candidate.message : void 0;
-  if (message && /(network|fetch failed|ECONNRESET|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNREFUSED)/i.test(message)) {
-    return true;
-  }
-  return false;
-}
-function isRetryableError(error, retryableStatusCodes) {
-  const status = extractStatus(error);
-  if (status === 401 || status === 403) {
-    return false;
-  }
-  if (typeof status === "number") {
-    return retryableStatusCodes.includes(status);
-  }
-  return isNetworkError(error);
-}
-function calculateRetryDelay(attempt, config) {
-  const delay = Math.min(
-    config.maxDelayMs,
-    config.initialDelayMs * config.backoffFactor ** attempt
-  );
-  return delay * (0.75 + Math.random() * 0.5);
-}
-async function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-async function withRetry(fn, retryConfig, signal) {
-  const config = {
-    maxRetries: retryConfig?.maxRetries ?? 3,
-    initialDelayMs: retryConfig?.initialDelayMs ?? 1e3,
-    maxDelayMs: retryConfig?.maxDelayMs ?? 6e4,
-    backoffFactor: retryConfig?.backoffFactor ?? 2,
-    retryableStatusCodes: retryConfig?.retryableStatusCodes ?? [500, 408, 429, 502, 503, 504]
-  };
-  let lastError;
-  for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
-    if (signal?.aborted) {
-      throw new Error(`Request aborted: ${signal.reason ?? "Unknown reason"}`);
-    }
-    try {
-      return await fn();
-    } catch (error) {
-      lastError = error;
-      if (attempt >= config.maxRetries) {
-        break;
-      }
-      if (!isRetryableError(error, config.retryableStatusCodes)) {
-        throw error;
-      }
-      const delay = calculateRetryDelay(attempt, config);
-      await sleep(delay);
-    }
-  }
-  throw lastError;
-}
-
 // src/evaluation/providers/claude-cli.ts
 import { spawn } from "node:child_process";
 import { randomUUID } from "node:crypto";
@@ -9054,10 +8632,10 @@ function extractToolCallsFromEvents(events) {
     }
   }
   const toolCalls = [];
-  for (const [id, { tool: tool2, input }] of starts) {
+  for (const [id, { tool, input }] of starts) {
     toolCalls.push(
       normalizeToolCall("pi-cli", {
-        tool: tool2,
+        tool,
         input,
         id: id.startsWith("anon-") ? void 0 : id,
         output: results.get(id)
@@ -10124,7 +9702,7 @@ import { readFile as readFile5 } from "node:fs/promises";
 import path20 from "node:path";
 
 // src/evaluation/providers/vscode/utils/time.ts
-function sleep2(ms) {
+function sleep(ms) {
   return new Promise((resolve) => {
     setTimeout(resolve, ms);
   });
@@ -10147,7 +9725,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
         }
         return false;
       }
-      await sleep2(pollInterval);
+      await sleep(pollInterval);
     }
   } catch (error) {
     if (error.code === "ENOENT") {
@@ -10173,7 +9751,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
         }
         return false;
       }
-      await sleep2(pollInterval);
+      await sleep(pollInterval);
     }
   }
   return false;
@@ -10202,7 +9780,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
         }
       }
       if (pending.size > 0) {
-        await sleep2(pollInterval);
+        await sleep(pollInterval);
       }
     }
   } catch (error) {
@@ -10230,7 +9808,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
           }
           return false;
         }
-        await sleep2(pollInterval);
+        await sleep(pollInterval);
       }
     }
   }
@@ -10326,7 +9904,7 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
     label: "open-workspace"
   });
   await raceSpawnError(workspaceChild);
-  await sleep2(100);
+  await sleep(100);
   const wakeupChatId = "wakeup";
   const chatArgs = [
     "-r",
@@ -10343,7 +9921,7 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
       console.error(`warning: Workspace readiness timeout after ${timeout}s`);
       return false;
     }
-    await sleep2(pollInterval * 1e3);
+    await sleep(pollInterval * 1e3);
   }
   return true;
 }
@@ -10371,7 +9949,7 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
       `VS Code workspace '${path22.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
     );
   }
-  await sleep2(500);
+  await sleep(500);
   const child = spawnVsCode(vscodeCmd, chatArgs, { label: "send-chat" });
   await raceSpawnError(child);
 }
@@ -10395,7 +9973,7 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
       `VS Code workspace '${path22.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
     );
   }
-  await sleep2(500);
+  await sleep(500);
   const child = spawnVsCode(vscodeCmd, chatArgs, { label: "send-batch-chat" });
   await raceSpawnError(child);
 }
@@ -16105,7 +15683,7 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
     return { tests: await loadTestsFromAgentSkills(evalFilePath) };
   }
   if (format === "typescript") {
-    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-4CFPGHGT.js");
+    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-E6MROJGR.js");
     return loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
   }
   const { tests, parsed, suiteWorkspacePath } = await loadTestsFromYaml(
@@ -16140,7 +15718,7 @@ async function loadTests(evalFilePath, repoRoot, options) {
     return loadTestsFromAgentSkills(evalFilePath);
   }
   if (format === "typescript") {
-    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-4CFPGHGT.js");
+    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-E6MROJGR.js");
     const suite = await loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
     return suite.tests;
   }
@@ -16496,7 +16074,8 @@ function parseWorkspaceConfig(raw, evalFileDir) {
   const workspacePath = typeof obj.path === "string" ? obj.path : void 0;
   const mode = explicitMode ?? (workspacePath ? "static" : void 0);
   const docker = parseDockerWorkspaceConfig(obj.docker);
-  if (!template && !isolation && !repos && !hooks && !mode && !workspacePath && !docker)
+  const env = parseWorkspaceEnvConfig(obj.env);
+  if (!template && !isolation && !repos && !hooks && !mode && !workspacePath && !docker && !env)
     return void 0;
   return {
     ...template !== void 0 && { template },
@@ -16505,7 +16084,19 @@ function parseWorkspaceConfig(raw, evalFileDir) {
     ...hooks !== void 0 && { hooks },
     ...mode !== void 0 && { mode },
     ...workspacePath !== void 0 && { path: workspacePath },
-    ...docker !== void 0 && { docker }
+    ...docker !== void 0 && { docker },
+    ...env !== void 0 && { env }
+  };
+}
+function parseWorkspaceEnvConfig(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  const required_commands = Array.isArray(obj.required_commands) ? obj.required_commands.filter((c) => typeof c === "string") : void 0;
+  const required_python_modules = Array.isArray(obj.required_python_modules) ? obj.required_python_modules.filter((m) => typeof m === "string") : void 0;
+  if (!required_commands?.length && !required_python_modules?.length) return void 0;
+  return {
+    ...required_commands?.length && { required_commands },
+    ...required_python_modules?.length && { required_python_modules }
   };
 }
 function parseDockerWorkspaceConfig(raw) {
@@ -16865,7 +16456,7 @@ async function runEvaluation(options) {
         if (!cliModel) {
           throw new Error('--grader-target "agentv" requires --model (e.g., "openai:gpt-5-mini")');
         }
-        const { AgentvProvider: AgentvProvider2 } = await import("./agentv-provider-TXM4UEUT.js");
+        const { AgentvProvider: AgentvProvider2 } = await import("./agentv-provider-MUIGGIP3.js");
         return new AgentvProvider2("agentv", { model: cliModel, temperature: 0 });
       }
       const overrideTarget = resolveTargetByName(cliGraderTarget);
@@ -17196,6 +16787,19 @@ async function runEvaluation(options) {
       await dockerSetup.pullImage();
       setupLog("Docker image pull complete");
     }
+    if (suiteWorkspace?.env) {
+      try {
+        await runPreflightChecks(suiteWorkspace.env, sharedWorkspacePath ?? void 0, setupLog);
+        setupLog("preflight checks passed");
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        if (sharedWorkspacePath && !useStaticWorkspace) {
+          await cleanupWorkspace(sharedWorkspacePath).catch(() => {
+          });
+        }
+        throw new Error(message);
+      }
+    }
     const suiteHooksEnabled = hooksEnabled(suiteWorkspace);
     const suiteBeforeAllHook = suiteWorkspace?.hooks?.before_all;
     if (sharedWorkspacePath && suiteHooksEnabled && hasHookCommand(suiteBeforeAllHook)) {
@@ -18220,7 +17824,7 @@ async function runEvalCase(options) {
       lastError = error;
       if (attempt + 1 < attemptBudget) {
         const delayMs = retryBackoffMs(attempt);
-        await sleep3(delayMs, signal);
+        await sleep2(delayMs, signal);
         attempt += 1;
         continue;
       }
@@ -19425,7 +19029,7 @@ function extractErrorMessage(error) {
 function retryBackoffMs(attempt) {
   return Math.min(2 ** attempt * 1e3, 3e4);
 }
-function sleep3(ms, signal) {
+function sleep2(ms, signal) {
   if (signal?.aborted) return Promise.resolve();
   return new Promise((resolve) => {
     const timer = setTimeout(resolve, ms);
@@ -19466,6 +19070,38 @@ function computeWeightedMean(entries) {
   }
   return totalWeight > 0 ? weightedSum / totalWeight : 0;
 }
+async function runPreflightChecks(env, cwd, log) {
+  const execFileAsync4 = promisify7(execFile3);
+  const missing = [];
+  for (const cmd of env.required_commands ?? []) {
+    log(`preflight: checking command "${cmd}"`);
+    try {
+      if (process.platform === "win32") {
+        await execFileAsync4("where", [cmd], { cwd });
+      } else {
+        await execFileAsync4("sh", ["-c", `command -v ${cmd}`], { cwd });
+      }
+    } catch {
+      missing.push(`command: ${cmd}`);
+    }
+  }
+  for (const mod of env.required_python_modules ?? []) {
+    log(`preflight: checking Python module "${mod}"`);
+    try {
+      await execFileAsync4("python3", ["-c", `import ${mod}`], { cwd });
+    } catch {
+      missing.push(`python module: ${mod}`);
+    }
+  }
+  if (missing.length > 0) {
+    throw new Error(
+      `Preflight checks failed \u2014 missing dependencies:
+${missing.map((m) => `  \u2022 ${m}`).join("\n")}
+
+Install the missing dependencies before running this eval.`
+    );
+  }
+}
 
 // src/evaluation/providers/function-provider.ts
 function createFunctionProvider(taskFn) {
@@ -19954,4 +19590,4 @@ export {
   loadTestById,
   loadEvalCaseById
 };
-//# sourceMappingURL=chunk-IXTJEXWN.js.map
+//# sourceMappingURL=chunk-EVEZQXIS.js.map