@agentv/core 4.25.1 → 4.25.2-next.1

package/dist/index.cjs

@@ -223,7 +223,7 @@ function computeTraceSummary(messages) {
 function explorationRatio(summary, explorationTools = DEFAULT_EXPLORATION_TOOLS) {
   if (summary.eventCount === 0) return void 0;
   const explorationCalls = explorationTools.reduce(
-    (sum, tool2) => sum + (summary.toolCalls[tool2] ?? 0),
+    (sum, tool) => sum + (summary.toolCalls[tool] ?? 0),
     0
   );
   return explorationCalls / summary.eventCount;
@@ -5187,8 +5187,17 @@ async function materializeContentForGrader(messages, getWorkDir) {
   }
   return result;
 }
+async function runScriptRaw(scriptPath, input, agentTimeoutMs, cwd, env) {
+  return typeof scriptPath === "string" ? execShellWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs, env }) : execFileWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs, env });
+}
 async function executeScript(scriptPath, input, agentTimeoutMs, cwd, env) {
-  const { stdout, stderr, exitCode } = typeof scriptPath === "string" ? await execShellWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs, env }) : await execFileWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs, env });
+  const { stdout, stderr, exitCode } = await runScriptRaw(
+    scriptPath,
+    input,
+    agentTimeoutMs,
+    cwd,
+    env
+  );
   if (exitCode !== 0) {
     const trimmedErr = formatStderr(stderr);
     throw new Error(
@@ -5306,6 +5315,8 @@ var init_code_grader = __esm({
         const env = proxyEnv || workspaceEnv ? { ...proxyEnv, ...workspaceEnv } : void 0;
         try {
           let stdout;
+          let exitCode = 0;
+          let execStderr = "";
           if (context2.dockerConfig) {
             const { DockerWorkspaceProvider: DockerWorkspaceProvider2 } = await Promise.resolve().then(() => (init_docker_workspace(), docker_workspace_exports));
             const dockerProvider = new DockerWorkspaceProvider2(context2.dockerConfig);
@@ -5314,31 +5325,40 @@ var init_code_grader = __esm({
               stdin: inputPayload,
               repoCheckouts: getRepoCheckoutTargets(context2.evalCase.workspace?.repos)
             });
-            if (result.exitCode !== 0) {
-              const trimmedErr = result.stderr.trim();
-              throw new Error(
-                trimmedErr.length > 0 ? `Code evaluator exited with code ${result.exitCode}: ${trimmedErr}` : `Code evaluator exited with code ${result.exitCode}`
-              );
-            }
+            exitCode = result.exitCode;
             stdout = result.stdout.trim();
+            execStderr = result.stderr;
           } else {
-            stdout = await executeScript(
+            const result = await runScriptRaw(
               this.command,
               inputPayload,
               this.agentTimeoutMs,
               this.cwd,
               env
             );
+            exitCode = result.exitCode;
+            stdout = result.stdout.trim();
+            execStderr = result.stderr;
           }
-          const parsed = parseJsonSafe(stdout);
-          const score = clampScore(typeof parsed?.score === "number" ? parsed.score : 0);
-          const assertions = Array.isArray(parsed?.assertions) ? parsed.assertions.filter(
+          const looksLikeJson = stdout.startsWith("{") || stdout.startsWith("[");
+          const hasStderr = execStderr.trim().length > 0;
+          if (exitCode !== 0 && (looksLikeJson || hasStderr)) {
+            const trimmedErr = formatStderr(execStderr);
+            throw new Error(
+              trimmedErr.length > 0 ? `Code evaluator exited with code ${exitCode}: ${trimmedErr}` : `Code evaluator exited with code ${exitCode}`
+            );
+          }
+          const rawParsed = parseJsonSafe(stdout);
+          const parsed = rawParsed != null && typeof rawParsed === "object" && !Array.isArray(rawParsed) ? rawParsed : void 0;
+          const passed = exitCode === 0;
+          const score = parsed != null ? clampScore(typeof parsed.score === "number" ? parsed.score : 0) : passed ? 1 : 0;
+          const assertions = parsed != null && Array.isArray(parsed?.assertions) ? parsed.assertions.filter(
             (a) => typeof a === "object" && a !== null && typeof a.text === "string"
           ).map((a) => ({
             text: String(a.text),
             passed: Boolean(a.passed),
             ...typeof a.evidence === "string" ? { evidence: a.evidence } : {}
-          })) : [];
+          })) : parsed == null ? [{ text: stdout.trim() || (passed ? "exit 0" : `exit ${exitCode}`), passed }] : [];
           const details = parsed?.details && typeof parsed.details === "object" && !Array.isArray(parsed.details) ? parsed.details : void 0;
           const proxyUsage = getProxyUsage?.();
           const graderRawRequest = {
@@ -5646,13 +5666,6 @@ function extractImageBlocks(messages) {
   }
   return images;
 }
-function toAiSdkImageParts(images) {
-  return images.map((img) => ({
-    type: "image",
-    image: img.source,
-    mediaType: img.media_type || void 0
-  }));
-}
 function resolveSandboxed(basePath, relativePath) {
   const resolved = import_node_path12.default.resolve(basePath, relativePath);
   if (!resolved.startsWith(basePath + import_node_path12.default.sep) && resolved !== basePath) {
@@ -5661,15 +5674,24 @@ function resolveSandboxed(basePath, relativePath) {
   return resolved;
 }
 function createFilesystemTools(workspacePath) {
-  return {
-    list_files: (0, import_ai.tool)({
+  return [
+    {
+      name: "list_files",
       description: "List files and directories at a relative path within the workspace. Returns names only (single level, no recursion).",
-      inputSchema: import_zod2.z.object({
-        path: import_zod2.z.string().describe('Relative path within workspace (use "." for root)').default(".")
-      }),
+      parameters: {
+        type: "object",
+        properties: {
+          path: {
+            type: "string",
+            description: 'Relative path within workspace (use "." for root)',
+            default: "."
+          }
+        }
+      },
       execute: async (input) => {
+        const args = input ?? {};
         try {
-          const resolved = resolveSandboxed(workspacePath, input.path);
+          const resolved = resolveSandboxed(workspacePath, args.path ?? ".");
           const entries = await import_promises12.default.readdir(resolved, { withFileTypes: true });
           return entries.map((e) => ({
             name: e.name,
@@ -5679,18 +5701,25 @@ function createFilesystemTools(workspacePath) {
           return { error: error instanceof Error ? error.message : String(error) };
         }
       }
-    }),
-    read_file: (0, import_ai.tool)({
+    },
+    {
+      name: "read_file",
       description: "Read the content of a file at a relative path within the workspace. Large files are truncated at 50KB.",
-      inputSchema: import_zod2.z.object({
-        path: import_zod2.z.string().describe("Relative path to file within workspace")
-      }),
+      parameters: {
+        type: "object",
+        properties: {
+          path: { type: "string", description: "Relative path to file within workspace" }
+        },
+        required: ["path"]
+      },
       execute: async (input) => {
+        const args = input ?? {};
+        const relPath = args.path ?? "";
         try {
-          const resolved = resolveSandboxed(workspacePath, input.path);
+          const resolved = resolveSandboxed(workspacePath, relPath);
           const stat14 = await import_promises12.default.stat(resolved);
           if (stat14.isDirectory()) {
-            return { error: `'${input.path}' is a directory, not a file` };
+            return { error: `'${relPath}' is a directory, not a file` };
           }
           const buffer = Buffer.alloc(Math.min(stat14.size, MAX_FILE_SIZE));
           const fd = await import_promises12.default.open(resolved, "r");
@@ -5706,19 +5735,29 @@ function createFilesystemTools(workspacePath) {
           return { error: error instanceof Error ? error.message : String(error) };
         }
       }
-    }),
-    search_files: (0, import_ai.tool)({
+    },
+    {
+      name: "search_files",
       description: "Search for a regex pattern across files in the workspace. Returns up to 20 matches. Skips binary files and node_modules/.git.",
-      inputSchema: import_zod2.z.object({
-        pattern: import_zod2.z.string().describe("Regex pattern to search for"),
-        path: import_zod2.z.string().describe('Relative path to search within (use "." for root)').default(".")
-      }),
+      parameters: {
+        type: "object",
+        properties: {
+          pattern: { type: "string", description: "Regex pattern to search for" },
+          path: {
+            type: "string",
+            description: 'Relative path to search within (use "." for root)',
+            default: "."
+          }
+        },
+        required: ["pattern"]
+      },
       execute: async (input) => {
+        const args = input ?? {};
         try {
-          const resolved = resolveSandboxed(workspacePath, input.path);
+          const resolved = resolveSandboxed(workspacePath, args.path ?? ".");
           let regex;
           try {
-            regex = new RegExp(input.pattern, "gi");
+            regex = new RegExp(args.pattern ?? "", "gi");
           } catch (regexErr) {
             return {
               error: `Invalid regex pattern: ${regexErr instanceof Error ? regexErr.message : String(regexErr)}`
@@ -5731,8 +5770,8 @@ function createFilesystemTools(workspacePath) {
           return { error: error instanceof Error ? error.message : String(error) };
         }
       }
-    })
-  };
+    }
+  ];
 }
 async function searchDirectory(dirPath, workspacePath, regex, matches) {
   if (matches.length >= MAX_SEARCH_MATCHES) return;
@@ -5772,14 +5811,13 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
     }
   }
 }
-var import_promises12, import_node_path12, import_ai, import_zod2, DEFAULT_MAX_STEPS, MAX_STEPS_LIMIT, MAX_FILE_SIZE, MAX_SEARCH_MATCHES, SEARCH_SKIP_DIRS, BINARY_EXTENSIONS, DEFAULT_GRADER_TEMPLATE, freeformEvaluationSchema, rubricCheckResultSchema, rubricEvaluationSchema, scoreRangeCheckResultSchema, scoreRangeEvaluationSchema, LlmGrader, ANSI_YELLOW7, ANSI_RESET8, warnedTemplateStrings;
+var import_promises12, import_node_path12, import_zod2, DEFAULT_MAX_STEPS, MAX_STEPS_LIMIT, MAX_FILE_SIZE, MAX_SEARCH_MATCHES, SEARCH_SKIP_DIRS, BINARY_EXTENSIONS, DEFAULT_GRADER_TEMPLATE, freeformEvaluationSchema, rubricCheckResultSchema, rubricEvaluationSchema, scoreRangeCheckResultSchema, scoreRangeEvaluationSchema, LlmGrader, ANSI_YELLOW7, ANSI_RESET8, warnedTemplateStrings;
 var init_llm_grader = __esm({
   "src/evaluation/graders/llm-grader.ts"() {
     "use strict";
     init_cjs_shims();
     import_promises12 = __toESM(require("fs/promises"), 1);
     import_node_path12 = __toESM(require("path"), 1);
-    import_ai = require("ai");
     import_zod2 = require("zod");
     init_content_preprocessor();
     init_content();
@@ -6095,18 +6133,15 @@ ${context2.toolCalls}`;
         }
       }
       // ---------------------------------------------------------------------------
-      // Built-in agent mode (agentv provider — AI SDK generateText with filesystem tools)
+      // Built-in agent mode (agentv provider — provider.invoke() with filesystem tools)
       // ---------------------------------------------------------------------------
       /**
-       * Built-in mode: Uses Vercel AI SDK generateText() with sandboxed filesystem tools.
+       * Built-in mode: drives the grader through provider.invoke() with the
+       * sandboxed filesystem tools and a step budget. The pi-ai-backed agentv
+       * provider runs the agent loop (tool call → tool execute → next model
+       * turn) until the model stops requesting tools or maxSteps is hit.
        */
       async evaluateBuiltIn(context2, graderProvider) {
-        const model = graderProvider.asLanguageModel?.();
-        if (!model) {
-          throw new Error(
-            `Grader provider '${graderProvider.targetName}' does not support asLanguageModel() \u2014 required for built-in agent mode`
-          );
-        }
         const workspacePath = context2.workspacePath;
         if (!workspacePath) {
           throw new Error(
@@ -6125,18 +6160,21 @@ ${context2.toolCalls}`;
           maxSteps: this.maxSteps
         };
         try {
-          const { text, steps } = await (0, import_ai.generateText)({
-            model,
-            system: systemPrompt,
-            prompt: userPrompt,
+          const response = await graderProvider.invoke({
+            question: userPrompt,
+            systemPrompt,
+            evalCaseId: context2.evalCase.id,
+            attempt: context2.attempt,
+            temperature: this.temperature ?? 0,
             tools: fsTools,
-            stopWhen: (0, import_ai.stepCountIs)(this.maxSteps),
-            temperature: this.temperature ?? 0
+            maxSteps: this.maxSteps
           });
-          const toolCallCount = steps.reduce((count, step) => count + (step.toolCalls?.length ?? 0), 0);
+          const text = extractLastAssistantContent2(response.output);
+          const stepCount = response.steps?.count ?? 1;
+          const toolCallCount = response.steps?.toolCallCount ?? 0;
           const details = {
             mode: "built-in",
-            steps: steps.length,
+            steps: stepCount,
             tool_calls: toolCallCount
           };
           return this.parseAgentResult(
@@ -6588,43 +6626,14 @@ ${outputSchema}`;
       }
       async generateStructuredResponse(options) {
         const { context: context2, graderProvider, systemPrompt, userPrompt, images } = options;
-        const model = graderProvider.asLanguageModel?.();
-        if (model) {
-          const modelOptions = {
-            ...this.maxOutputTokens ? { maxTokens: this.maxOutputTokens } : {},
-            ...typeof this.temperature === "number" ? { temperature: this.temperature } : {}
-          };
-          const hasImages = images && images.length > 0;
-          const result = hasImages ? await (0, import_ai.generateText)({
-            model,
-            system: systemPrompt,
-            messages: [
-              {
-                role: "user",
-                content: [
-                  { type: "text", text: userPrompt },
-                  ...toAiSdkImageParts(images)
-                ]
-              }
-            ],
-            ...modelOptions
-          }) : await (0, import_ai.generateText)({
-            model,
-            system: systemPrompt,
-            prompt: userPrompt,
-            ...modelOptions
-          });
-          const rawUsage = result.usage;
-          const tokenUsage = rawUsage?.inputTokens != null && rawUsage?.outputTokens != null ? { input: rawUsage.inputTokens, output: rawUsage.outputTokens } : void 0;
-          return { text: result.text, tokenUsage };
-        }
         const response = await graderProvider.invoke({
           question: userPrompt,
           systemPrompt,
           evalCaseId: context2.evalCase.id,
           attempt: context2.attempt,
           maxOutputTokens: this.maxOutputTokens,
-          temperature: this.temperature
+          temperature: this.temperature,
+          ...images && images.length > 0 ? { images } : {}
         });
         return {
           text: extractLastAssistantContent2(response.output),
@@ -6640,12 +6649,11 @@ ${outputSchema}`;
 });
 
 // src/evaluation/graders/composite.ts
-var import_ai2, DEFAULT_COMPOSITE_AGGREGATOR_PROMPT, CompositeGrader;
+var DEFAULT_COMPOSITE_AGGREGATOR_PROMPT, CompositeGrader;
 var init_composite = __esm({
   "src/evaluation/graders/composite.ts"() {
     "use strict";
     init_cjs_shims();
-    import_ai2 = require("ai");
     init_types2();
     init_code_grader();
     init_llm_grader();
@@ -6888,25 +6896,6 @@ Return a JSON object with: score (0.0-1.0), verdict (pass/fail), and reasoning.`
           target: graderProvider.targetName
         };
         try {
-          const model = graderProvider.asLanguageModel?.();
-          if (model) {
-            const { text } = await (0, import_ai2.generateText)({
-              model,
-              system: systemPrompt,
-              prompt: userPrompt
-            });
-            const data2 = freeformEvaluationSchema.parse(parseJsonFromText(text));
-            const score2 = clampScore(data2.score);
-            const assertions2 = Array.isArray(data2.assertions) ? data2.assertions.slice(0, 8) : [];
-            return {
-              score: score2,
-              verdict: scoreToVerdict(score2),
-              assertions: assertions2,
-              expectedAspectCount: Math.max(assertions2.length, 1),
-              graderRawRequest,
-              scores
-            };
-          }
           const response = await graderProvider.invoke({
             question: userPrompt,
             systemPrompt,
@@ -8689,115 +8678,254 @@ var init_graders2 = __esm({
   }
 });
 
-// src/evaluation/providers/agentv-provider.ts
-var agentv_provider_exports = {};
-__export(agentv_provider_exports, {
-  AgentvProvider: () => AgentvProvider
-});
-function parseModelString(model) {
-  const colonIndex = model.indexOf(":");
-  if (colonIndex === -1) {
-    throw new Error(
-      `Invalid model string "${model}". Expected format "provider:model" (e.g., "openai:gpt-5-mini")`
+// src/evaluation/providers/llm-providers.ts
+function buildAzureBaseUrl(input) {
+  const trimmed = input.replace(/\/+$/, "");
+  if (trimmed.endsWith("/openai/v1")) return trimmed;
+  if (trimmed.endsWith("/openai")) return `${trimmed}/v1`;
+  return `${trimmed}/openai/v1`;
+}
+async function invokePiAi(options) {
+  const { model, apiKey, request, defaults, retryConfig, providerOptions } = options;
+  const tools = request.tools && request.tools.length > 0 ? request.tools : void 0;
+  const maxSteps = tools ? Math.max(1, request.maxSteps ?? 1) : 1;
+  const { systemPrompt, messages } = chatPromptToPiContext(buildChatPrompt(request));
+  if (request.images && request.images.length > 0) {
+    attachImagesToLastUserMessage(messages, request.images);
+  }
+  const piTools = tools ? tools.map((t) => ({
+    name: t.name,
+    description: t.description,
+    parameters: t.parameters
+  })) : void 0;
+  const ctx = { systemPrompt, messages, ...piTools ? { tools: piTools } : {} };
+  const { temperature, maxOutputTokens } = resolveModelSettings(request, defaults);
+  const callOptions = {
+    ...apiKey !== void 0 ? { apiKey } : {},
+    temperature,
+    ...maxOutputTokens !== void 0 ? { maxTokens: maxOutputTokens } : {},
+    signal: request.signal,
+    ...providerOptions ?? {}
+  };
+  const startTime = (/* @__PURE__ */ new Date()).toISOString();
+  const startMs = Date.now();
+  const aggregateUsage = { input: 0, output: 0, cacheRead: 0, cost: 0 };
+  let stepCount = 0;
+  let toolCallCount = 0;
+  let result = await withRetry(
+    () => (0, import_pi_ai.complete)(model, ctx, callOptions),
+    retryConfig,
+    request.signal
+  );
+  ctx.messages.push(result);
+  stepCount = 1;
+  accumulateUsage(aggregateUsage, result.usage);
+  while (tools) {
+    const calls = result.content.filter(
+      (b) => b.type === "toolCall"
+    );
+    if (calls.length === 0) break;
+    if (stepCount >= maxSteps) break;
+    toolCallCount += calls.length;
+    for (const call of calls) {
+      const tool = tools.find((t) => t.name === call.name);
+      let output;
+      let isError = false;
+      try {
+        if (!tool) {
+          throw new Error(`pi-ai adapter: model called unknown tool '${call.name}'`);
+        }
+        output = await tool.execute(call.arguments);
+      } catch (err) {
+        output = err instanceof Error ? err.message : String(err);
+        isError = true;
+      }
+      ctx.messages.push({
+        role: "toolResult",
+        toolCallId: call.id,
+        toolName: call.name,
+        content: [
+          { type: "text", text: typeof output === "string" ? output : JSON.stringify(output) }
+        ],
+        isError,
+        timestamp: Date.now()
+      });
+    }
+    result = await withRetry(
+      () => (0, import_pi_ai.complete)(model, ctx, callOptions),
+      retryConfig,
+      request.signal
     );
+    ctx.messages.push(result);
+    stepCount += 1;
+    accumulateUsage(aggregateUsage, result.usage);
   }
-  return {
-    provider: model.slice(0, colonIndex),
-    modelName: model.slice(colonIndex + 1)
-  };
+  const endTime = (/* @__PURE__ */ new Date()).toISOString();
+  const durationMs = Date.now() - startMs;
+  return mapPiResponse(result, {
+    durationMs,
+    startTime,
+    endTime,
+    aggregateUsage,
+    steps: tools ? { count: stepCount, toolCallCount } : void 0
+  });
 }
-function createLanguageModel(modelString) {
-  const { provider, modelName } = parseModelString(modelString);
-  switch (provider) {
-    case "openai":
-      return (0, import_openai.createOpenAI)()(modelName);
-    case "anthropic":
-      return (0, import_anthropic.createAnthropic)()(modelName);
-    case "azure":
-      return (0, import_azure.createAzure)().chat(modelName);
-    case "google":
-      return (0, import_google.createGoogleGenerativeAI)()(modelName);
-    default:
+function accumulateUsage(agg, u) {
+  agg.input += u.input;
+  agg.output += u.output;
+  agg.cacheRead += u.cacheRead;
+  agg.cost += u.cost.total;
+}
+function resolvePiModel(args) {
+  const { providerName, apiId, modelId, baseUrl } = args;
+  let model;
+  try {
+    model = (0, import_pi_ai.getModel)(providerName, modelId);
+  } catch {
+    model = void 0;
+  }
+  if (!model) {
+    const fallbackBaseUrl = baseUrl ?? defaultBaseUrlFor(providerName);
+    if (!fallbackBaseUrl) {
       throw new Error(
-        `Unsupported AI SDK provider "${provider}" in model string "${modelString}". Supported providers: openai, anthropic, azure, google`
+        `pi-ai adapter cannot resolve a baseUrl for provider '${providerName}' / model '${modelId}'. Either set the target's baseUrl/endpoint or use a model id pi-ai recognizes.`
       );
-  }
-}
-var import_anthropic, import_azure, import_google, import_openai, AgentvProvider;
-var init_agentv_provider = __esm({
-  "src/evaluation/providers/agentv-provider.ts"() {
-    "use strict";
-    init_cjs_shims();
-    import_anthropic = require("@ai-sdk/anthropic");
-    import_azure = require("@ai-sdk/azure");
-    import_google = require("@ai-sdk/google");
-    import_openai = require("@ai-sdk/openai");
-    AgentvProvider = class {
-      id;
-      kind = "agentv";
-      targetName;
-      model;
-      constructor(targetName, config) {
-        this.id = `agentv:${targetName}`;
-        this.targetName = targetName;
-        this.model = createLanguageModel(config.model);
-      }
-      /**
-       * Direct invoke is not supported for the agentv provider.
-       * Use asLanguageModel() with generateText() instead.
-       */
-      async invoke(_request) {
-        throw new Error(
-          "AgentvProvider does not support direct invoke(). Use asLanguageModel() with generateText() instead."
-        );
-      }
-      /**
-       * Returns the resolved AI SDK LanguageModel for use with generateText/generateObject.
-       */
-      asLanguageModel() {
-        return this.model;
-      }
+    }
+    model = {
+      id: modelId,
+      name: modelId,
+      api: apiId,
+      provider: providerName,
+      baseUrl: fallbackBaseUrl,
+      reasoning: false,
+      input: ["text"],
+      cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+      contextWindow: 128e3,
+      maxTokens: 16384
     };
   }
-});
-
-// src/evaluation/providers/ai-sdk.ts
-function buildAzureOptions(config) {
-  const options = {
-    apiKey: config.apiKey,
-    apiVersion: config.version,
-    // Chat completions still use deployment-scoped Azure URLs for compatibility
-    // with existing deployments. Responses API should use the SDK's v1 path.
-    useDeploymentBasedUrls: config.apiFormat !== "responses"
-  };
-  const baseURL = normalizeAzureBaseUrl(config.resourceName);
-  if (baseURL) {
-    options.baseURL = baseURL;
-  } else {
-    options.resourceName = config.resourceName;
+  if (model.api !== apiId) {
+    model = { ...model, api: apiId };
+  }
+  if (baseUrl) {
+    model = { ...model, baseUrl };
   }
-  return options;
+  return model;
 }
-function normalizeAzureBaseUrl(resourceName) {
-  const trimmed = resourceName.trim();
-  if (!/^https?:\/\//i.test(trimmed)) {
-    return void 0;
+function defaultBaseUrlFor(providerName) {
+  if (providerName === "openai") return "https://api.openai.com/v1";
+  if (providerName === "openrouter") return "https://openrouter.ai/api/v1";
+  return void 0;
+}
+function chatPromptToPiContext(chatPrompt) {
+  const systemSegments = [];
+  const messages = [];
+  const now = Date.now();
+  for (const message of chatPrompt) {
+    if (message.role === "system") {
+      systemSegments.push(message.content);
+      continue;
+    }
+    if (message.role === "user") {
+      messages.push({ role: "user", content: message.content, timestamp: now });
+      continue;
+    }
+    if (message.role === "assistant") {
+      messages.push({
+        role: "assistant",
+        content: [{ type: "text", text: message.content }],
+        api: "",
+        provider: "",
+        model: "",
+        usage: {
+          input: 0,
+          output: 0,
+          cacheRead: 0,
+          cacheWrite: 0,
+          totalTokens: 0,
+          cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }
+        },
+        stopReason: "stop",
+        timestamp: now
+      });
+      continue;
+    }
+    if (message.role === "tool" || message.role === "function") {
+      const prefix = message.name ? `@[${message.name}]: ` : "@[Tool]: ";
+      messages.push({
+        role: "assistant",
+        content: [{ type: "text", text: `${prefix}${message.content}` }],
+        api: "",
+        provider: "",
+        model: "",
+        usage: {
+          input: 0,
+          output: 0,
+          cacheRead: 0,
+          cacheWrite: 0,
+          totalTokens: 0,
+          cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }
+        },
+        stopReason: "stop",
+        timestamp: now
+      });
+      continue;
+    }
+    throw new Error(`pi-ai adapter received unsupported message role '${message.role}'.`);
   }
-  const withoutSlash = trimmed.replace(/\/+$/, "");
-  const normalized = withoutSlash.endsWith("/openai") ? withoutSlash : `${withoutSlash}/openai`;
-  return normalized;
+  return {
+    systemPrompt: systemSegments.length > 0 ? systemSegments.join("\n\n") : void 0,
+    messages
+  };
 }
-function buildAnthropicProviderOptions(defaults) {
-  if (defaults.thinkingBudget === void 0) {
-    return void 0;
+function attachImagesToLastUserMessage(messages, images) {
+  if (!images || images.length === 0) return;
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const m = messages[i];
+    if (m.role !== "user") continue;
+    const text = typeof m.content === "string" ? m.content : "";
+    messages[i] = {
+      ...m,
+      content: [
+        ...text ? [{ type: "text", text }] : [],
+        ...images.map((img) => ({
+          type: "image",
+          data: img.source,
+          mimeType: img.media_type
+        }))
+      ]
+    };
+    return;
   }
+  messages.push({
+    role: "user",
+    content: images.map((img) => ({
+      type: "image",
+      data: img.source,
+      mimeType: img.media_type
+    })),
+    timestamp: Date.now()
+  });
+}
+function mapPiResponse(result, timing) {
+  const text = result.content.filter((b) => b.type === "text").map((b) => b.text).join("");
+  const cached = timing.aggregateUsage.cacheRead > 0 ? timing.aggregateUsage.cacheRead : void 0;
+  const tokenUsage = {
+    input: timing.aggregateUsage.input,
+    output: timing.aggregateUsage.output,
+    ...cached !== void 0 ? { cached } : {}
+  };
+  const costUsd = timing.aggregateUsage.cost > 0 ? timing.aggregateUsage.cost : void 0;
   return {
-    anthropic: {
-      thinking: {
-        type: "enabled",
-        budgetTokens: defaults.thinkingBudget
-      }
-    }
+    raw: result,
+    usage: toJsonObject(result.usage),
+    output: [{ role: "assistant", content: text }],
+    tokenUsage,
+    ...costUsd !== void 0 ? { costUsd } : {},
+    durationMs: timing.durationMs,
+    startTime: timing.startTime,
+    endTime: timing.endTime,
+    ...timing.steps ? { steps: timing.steps } : {}
   };
 }
 function buildChatPrompt(request) {
@@ -8812,92 +8940,21 @@ function buildChatPrompt(request) {
   }
   const systemContent = resolveSystemContent(request);
   const userContent = request.question.trim();
-  const prompt = [
+  return [
     { role: "system", content: systemContent },
     { role: "user", content: userContent }
   ];
-  return prompt;
 }
 function resolveSystemContent(request) {
-  const systemSegments = [];
   if (request.systemPrompt && request.systemPrompt.trim().length > 0) {
-    systemSegments.push(request.systemPrompt.trim());
-  } else {
-    systemSegments.push(DEFAULT_SYSTEM_PROMPT);
+    return request.systemPrompt.trim();
   }
-  return systemSegments.join("\n\n");
-}
-function toModelMessages(chatPrompt) {
-  return chatPrompt.map((message) => {
-    if (message.role === "tool" || message.role === "function") {
-      const prefix = message.name ? `@[${message.name}]: ` : "@[Tool]: ";
-      return {
-        role: "assistant",
-        content: `${prefix}${message.content}`
-      };
-    }
-    if (message.role === "assistant" || message.role === "system" || message.role === "user") {
-      return {
-        role: message.role,
-        content: message.content
-      };
-    }
-    return {
-      role: "user",
-      content: message.content
-    };
-  });
+  return DEFAULT_SYSTEM_PROMPT;
 }
 function resolveModelSettings(request, defaults) {
-  const temperature = request.temperature ?? defaults.temperature;
-  const maxOutputTokens = request.maxOutputTokens ?? defaults.maxOutputTokens;
   return {
-    temperature,
-    maxOutputTokens
-  };
-}
-async function invokeModel(options) {
-  const { model, request, defaults, retryConfig, providerOptions } = options;
-  const chatPrompt = buildChatPrompt(request);
-  const { temperature, maxOutputTokens } = resolveModelSettings(request, defaults);
-  const startTime = (/* @__PURE__ */ new Date()).toISOString();
-  const startMs = Date.now();
-  const result = await withRetry(
-    () => (0, import_ai3.generateText)({
-      model,
-      messages: toModelMessages(chatPrompt),
-      temperature,
-      maxOutputTokens,
-      maxRetries: 0,
-      abortSignal: request.signal,
-      ...providerOptions ? { providerOptions } : {}
-    }),
-    retryConfig,
-    request.signal
-  );
-  const endTime = (/* @__PURE__ */ new Date()).toISOString();
-  const durationMs = Date.now() - startMs;
-  return mapResponse(result, { durationMs, startTime, endTime });
-}
-function mapResponse(result, timing) {
-  const content = result.text ?? "";
-  const rawUsage = result.totalUsage ?? result.usage;
-  const reasoning = rawUsage?.outputTokenDetails?.reasoningTokens ?? void 0;
-  const cached = rawUsage?.inputTokenDetails?.cacheReadTokens ?? void 0;
-  const tokenUsage = rawUsage?.inputTokens != null && rawUsage?.outputTokens != null ? {
-    input: rawUsage.inputTokens,
-    output: rawUsage.outputTokens,
-    ...reasoning != null ? { reasoning } : {},
-    ...cached != null ? { cached } : {}
-  } : void 0;
-  return {
-    raw: result,
-    usage: toJsonObject(rawUsage),
-    output: [{ role: "assistant", content }],
-    tokenUsage,
-    durationMs: timing?.durationMs,
-    startTime: timing?.startTime,
-    endTime: timing?.endTime
+    temperature: request.temperature ?? defaults.temperature,
+    maxOutputTokens: request.maxOutputTokens ?? defaults.maxOutputTokens
   };
 }
 function toJsonObject(value) {
@@ -8911,9 +8968,7 @@ function toJsonObject(value) {
   }
 }
 function extractStatus(error) {
-  if (!error || typeof error !== "object") {
-    return void 0;
-  }
+  if (!error || typeof error !== "object") return void 0;
   const candidate = error;
   const directStatus = candidate.status ?? candidate.statusCode;
   if (typeof directStatus === "number" && Number.isFinite(directStatus)) {
@@ -8928,21 +8983,15 @@ function extractStatus(error) {
     const match = message.match(/HTTP\s+(\d{3})/i);
     if (match) {
       const parsed = Number.parseInt(match[1], 10);
-      if (Number.isFinite(parsed)) {
-        return parsed;
-      }
+      if (Number.isFinite(parsed)) return parsed;
     }
   }
   return void 0;
 }
 function isNetworkError(error) {
-  if (!error || typeof error !== "object") {
-    return false;
-  }
+  if (!error || typeof error !== "object") return false;
   const candidate = error;
-  if (candidate.name === "AbortError") {
-    return false;
-  }
+  if (candidate.name === "AbortError") return false;
   const code = candidate.code;
   if (typeof code === "string" && /^E(AI|CONN|HOST|NET|PIPE|TIME|REFUSED|RESET)/i.test(code)) {
     return true;
@@ -8955,12 +9004,8 @@ function isNetworkError(error) {
 }
 function isRetryableError(error, retryableStatusCodes) {
   const status = extractStatus(error);
-  if (status === 401 || status === 403) {
-    return false;
-  }
-  if (typeof status === "number") {
-    return retryableStatusCodes.includes(status);
-  }
+  if (status === 401 || status === 403) return false;
+  if (typeof status === "number") return retryableStatusCodes.includes(status);
   return isNetworkError(error);
 }
 function calculateRetryDelay(attempt, config) {
@@ -8990,195 +9035,266 @@ async function withRetry(fn, retryConfig, signal) {
       return await fn();
     } catch (error) {
       lastError = error;
-      if (attempt >= config.maxRetries) {
-        break;
-      }
-      if (!isRetryableError(error, config.retryableStatusCodes)) {
-        throw error;
-      }
+      if (attempt >= config.maxRetries) break;
+      if (!isRetryableError(error, config.retryableStatusCodes)) throw error;
       const delay = calculateRetryDelay(attempt, config);
       await sleep(delay);
     }
   }
   throw lastError;
 }
-var import_anthropic2, import_azure2, import_google2, import_openai2, import_ai_sdk_provider, import_ai3, DEFAULT_SYSTEM_PROMPT, OpenAIProvider, AzureProvider, OpenRouterProvider, AnthropicProvider, GeminiProvider;
-var init_ai_sdk = __esm({
-  "src/evaluation/providers/ai-sdk.ts"() {
+var import_pi_ai, DEFAULT_SYSTEM_PROMPT, OpenAIProvider, OpenRouterProvider, AnthropicProvider, GeminiProvider, AzureProvider;
+var init_llm_providers = __esm({
+  "src/evaluation/providers/llm-providers.ts"() {
     "use strict";
     init_cjs_shims();
-    import_anthropic2 = require("@ai-sdk/anthropic");
-    import_azure2 = require("@ai-sdk/azure");
-    import_google2 = require("@ai-sdk/google");
-    import_openai2 = require("@ai-sdk/openai");
-    import_ai_sdk_provider = require("@openrouter/ai-sdk-provider");
-    import_ai3 = require("ai");
+    import_pi_ai = require("@mariozechner/pi-ai");
+    (0, import_pi_ai.registerBuiltInApiProviders)();
     DEFAULT_SYSTEM_PROMPT = "You are a careful assistant. Follow all provided instructions and do not fabricate results.";
     OpenAIProvider = class {
-      constructor(targetName, config) {
-        this.config = config;
-        this.id = `openai:${targetName}`;
-        this.targetName = targetName;
-        this.defaults = {
-          temperature: config.temperature,
-          maxOutputTokens: config.maxOutputTokens
-        };
-        this.retryConfig = config.retry;
-        const openai = (0, import_openai2.createOpenAI)({
-          apiKey: config.apiKey,
-          baseURL: config.baseURL
-        });
-        this.model = config.apiFormat === "responses" ? openai(config.model) : openai.chat(config.model);
-      }
       id;
       kind = "openai";
       targetName;
-      model;
+      piModel;
       defaults;
       retryConfig;
-      async invoke(request) {
-        return invokeModel({
-          model: this.model,
-          request,
-          defaults: this.defaults,
-          retryConfig: this.retryConfig
-        });
-      }
-      asLanguageModel() {
-        return this.model;
-      }
-    };
-    AzureProvider = class {
+      apiKey;
       constructor(targetName, config) {
-        this.config = config;
-        this.id = `azure:${targetName}`;
+        this.id = `openai:${targetName}`;
         this.targetName = targetName;
+        this.apiKey = config.apiKey;
         this.defaults = {
           temperature: config.temperature,
           maxOutputTokens: config.maxOutputTokens
         };
         this.retryConfig = config.retry;
-        const azure = (0, import_azure2.createAzure)(buildAzureOptions(config));
-        this.model = config.apiFormat === "responses" ? azure(config.deploymentName) : azure.chat(config.deploymentName);
+        this.piModel = resolvePiModel({
+          providerName: "openai",
+          apiId: config.apiFormat === "responses" ? "openai-responses" : "openai-completions",
+          modelId: config.model,
+          baseUrl: config.baseURL
+        });
       }
-      id;
-      kind = "azure";
-      targetName;
-      model;
-      defaults;
-      retryConfig;
       async invoke(request) {
-        return invokeModel({
-          model: this.model,
+        return invokePiAi({
+          model: this.piModel,
+          apiKey: this.apiKey,
           request,
           defaults: this.defaults,
           retryConfig: this.retryConfig
         });
       }
-      asLanguageModel() {
-        return this.model;
-      }
     };
     OpenRouterProvider = class {
+      id;
+      kind = "openrouter";
+      targetName;
+      piModel;
+      defaults;
+      retryConfig;
+      apiKey;
       constructor(targetName, config) {
-        this.config = config;
         this.id = `openrouter:${targetName}`;
         this.targetName = targetName;
+        this.apiKey = config.apiKey;
         this.defaults = {
           temperature: config.temperature,
           maxOutputTokens: config.maxOutputTokens
         };
         this.retryConfig = config.retry;
-        const openrouter = (0, import_ai_sdk_provider.createOpenRouter)({
-          apiKey: config.apiKey
+        this.piModel = resolvePiModel({
+          providerName: "openrouter",
+          apiId: "openai-completions",
+          modelId: config.model,
+          baseUrl: "https://openrouter.ai/api/v1"
         });
-        this.model = openrouter(config.model);
       }
-      id;
-      kind = "openrouter";
-      targetName;
-      model;
-      defaults;
-      retryConfig;
       async invoke(request) {
-        return invokeModel({
-          model: this.model,
+        return invokePiAi({
+          model: this.piModel,
+          apiKey: this.apiKey,
           request,
           defaults: this.defaults,
           retryConfig: this.retryConfig
         });
       }
-      asLanguageModel() {
-        return this.model;
-      }
     };
     AnthropicProvider = class {
+      id;
+      kind = "anthropic";
+      targetName;
+      piModel;
+      defaults;
+      retryConfig;
+      apiKey;
+      thinkingBudget;
       constructor(targetName, config) {
-        this.config = config;
         this.id = `anthropic:${targetName}`;
         this.targetName = targetName;
+        this.apiKey = config.apiKey;
+        this.thinkingBudget = config.thinkingBudget;
         this.defaults = {
           temperature: config.temperature,
           maxOutputTokens: config.maxOutputTokens,
           thinkingBudget: config.thinkingBudget
         };
         this.retryConfig = config.retry;
-        const anthropic = (0, import_anthropic2.createAnthropic)({
-          apiKey: config.apiKey
+        this.piModel = resolvePiModel({
+          providerName: "anthropic",
+          apiId: "anthropic-messages",
+          modelId: config.model
         });
-        this.model = anthropic(config.model);
       }
-      id;
-      kind = "anthropic";
-      targetName;
-      model;
-      defaults;
-      retryConfig;
       async invoke(request) {
-        const providerOptions = buildAnthropicProviderOptions(this.defaults);
-        return invokeModel({
-          model: this.model,
+        const providerOptions = this.thinkingBudget !== void 0 ? { thinkingEnabled: true, thinkingBudgetTokens: this.thinkingBudget } : void 0;
+        return invokePiAi({
+          model: this.piModel,
+          apiKey: this.apiKey,
           request,
           defaults: this.defaults,
           retryConfig: this.retryConfig,
-          providerOptions
+          ...providerOptions ? { providerOptions } : {}
         });
       }
-      asLanguageModel() {
-        return this.model;
-      }
     };
     GeminiProvider = class {
+      id;
+      kind = "gemini";
+      targetName;
+      piModel;
+      defaults;
+      retryConfig;
+      apiKey;
       constructor(targetName, config) {
-        this.config = config;
         this.id = `gemini:${targetName}`;
         this.targetName = targetName;
+        this.apiKey = config.apiKey;
         this.defaults = {
           temperature: config.temperature,
           maxOutputTokens: config.maxOutputTokens
         };
         this.retryConfig = config.retry;
-        const google = (0, import_google2.createGoogleGenerativeAI)({
-          apiKey: config.apiKey
+        this.piModel = resolvePiModel({
+          providerName: "google",
+          apiId: "google-generative-ai",
+          modelId: config.model
+        });
+      }
+      async invoke(request) {
+        return invokePiAi({
+          model: this.piModel,
+          apiKey: this.apiKey,
+          request,
+          defaults: this.defaults,
+          retryConfig: this.retryConfig
         });
-        this.model = google(config.model);
       }
+    };
+    AzureProvider = class {
       id;
-      kind = "gemini";
+      kind = "azure";
       targetName;
-      model;
+      piModel;
       defaults;
       retryConfig;
+      apiKey;
+      providerOptions;
+      constructor(targetName, config) {
+        this.id = `azure:${targetName}`;
+        this.targetName = targetName;
+        this.apiKey = config.apiKey;
+        this.defaults = {
+          temperature: config.temperature,
+          maxOutputTokens: config.maxOutputTokens
+        };
+        this.retryConfig = config.retry;
+        const trimmed = config.resourceName.trim();
+        const isFullUrl = /^https?:\/\//i.test(trimmed);
+        const baseUrl = isFullUrl ? buildAzureBaseUrl(trimmed) : void 0;
+        this.providerOptions = {
+          ...baseUrl ? { azureBaseUrl: baseUrl } : { azureResourceName: trimmed },
+          ...config.version ? { azureApiVersion: config.version } : {}
+        };
+        this.piModel = resolvePiModel({
+          providerName: "azure-openai-responses",
+          apiId: "azure-openai-responses",
+          // The "model id" for Azure is the deployment name.
+          modelId: config.deploymentName,
+          ...baseUrl ? { baseUrl } : {}
+        });
+      }
       async invoke(request) {
-        return invokeModel({
-          model: this.model,
+        return invokePiAi({
+          model: this.piModel,
+          apiKey: this.apiKey,
           request,
           defaults: this.defaults,
-          retryConfig: this.retryConfig
+          retryConfig: this.retryConfig,
+          providerOptions: this.providerOptions
         });
       }
-      asLanguageModel() {
-        return this.model;
+    };
+  }
+});
+
+// src/evaluation/providers/agentv-provider.ts
+var agentv_provider_exports = {};
+__export(agentv_provider_exports, {
+  AgentvProvider: () => AgentvProvider
+});
+function parseAgentvModel(model) {
+  const colonIndex = model.indexOf(":");
+  if (colonIndex === -1) {
+    throw new Error(
+      `Invalid agentv model "${model}". Expected "provider:model" (e.g., "openai:gpt-5-mini").`
+    );
+  }
+  const provider = model.slice(0, colonIndex);
+  const modelId = model.slice(colonIndex + 1);
+  switch (provider) {
+    case "openai":
+      return { providerName: "openai", apiId: "openai-completions", modelId };
+    case "anthropic":
+      return { providerName: "anthropic", apiId: "anthropic-messages", modelId };
+    case "azure":
+      return {
+        providerName: "azure-openai-responses",
+        apiId: "azure-openai-responses",
+        modelId
+      };
+    case "google":
+      return { providerName: "google", apiId: "google-generative-ai", modelId };
+    default:
+      throw new Error(
+        `Unsupported agentv provider "${provider}" in "${model}". Supported: openai, anthropic, azure, google.`
+      );
+  }
+}
+var AgentvProvider;
+var init_agentv_provider = __esm({
+  "src/evaluation/providers/agentv-provider.ts"() {
+    "use strict";
+    init_cjs_shims();
+    init_llm_providers();
+    AgentvProvider = class {
+      id;
+      kind = "agentv";
+      targetName;
+      piModel;
+      defaults;
+      constructor(targetName, config) {
+        this.id = `agentv:${targetName}`;
+        this.targetName = targetName;
+        const { providerName, apiId, modelId } = parseAgentvModel(config.model);
+        this.piModel = resolvePiModel({ providerName, apiId, modelId });
+        this.defaults = { temperature: config.temperature };
+      }
+      async invoke(request) {
+        return invokePiAi({
+          model: this.piModel,
+          request,
+          defaults: this.defaults
+        });
       }
     };
   }
@@ -13381,10 +13497,10 @@ function extractToolCallsFromEvents(events) {
     }
   }
   const toolCalls = [];
-  for (const [id, { tool: tool2, input }] of starts) {
+  for (const [id, { tool, input }] of starts) {
     toolCalls.push(
       normalizeToolCall("pi-cli", {
-        tool: tool2,
+        tool,
         input,
         id: id.startsWith("anon-") ? void 0 : id,
         output: results.get(id)
@@ -17765,7 +17881,6 @@ var init_providers = __esm({
     "use strict";
     init_cjs_shims();
     init_agentv_provider();
-    init_ai_sdk();
     init_claude_cli();
     init_claude_sdk();
     init_cli();
@@ -17773,6 +17888,7 @@ var init_providers = __esm({
     init_copilot_cli();
     init_copilot_log();
     init_copilot_sdk();
+    init_llm_providers();
     init_mock();
     init_pi_cli();
     init_pi_coding_agent();
@@ -19799,6 +19915,19 @@ async function runEvaluation(options) {
       await dockerSetup.pullImage();
       setupLog("Docker image pull complete");
     }
+    if (suiteWorkspace?.env) {
+      try {
+        await runPreflightChecks(suiteWorkspace.env, sharedWorkspacePath ?? void 0, setupLog);
+        setupLog("preflight checks passed");
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        if (sharedWorkspacePath && !useStaticWorkspace) {
+          await cleanupWorkspace(sharedWorkspacePath).catch(() => {
+          });
+        }
+        throw new Error(message);
+      }
+    }
     const suiteHooksEnabled = hooksEnabled(suiteWorkspace);
     const suiteBeforeAllHook = suiteWorkspace?.hooks?.before_all;
     if (sharedWorkspacePath && suiteHooksEnabled && hasHookCommand(suiteBeforeAllHook)) {
@@ -22069,6 +22198,38 @@ function computeWeightedMean(entries) {
   }
   return totalWeight > 0 ? weightedSum / totalWeight : 0;
 }
+async function runPreflightChecks(env, cwd, log) {
+  const execFileAsync5 = (0, import_node_util7.promisify)(import_node_child_process11.execFile);
+  const missing = [];
+  for (const cmd of env.required_commands ?? []) {
+    log(`preflight: checking command "${cmd}"`);
+    try {
+      if (process.platform === "win32") {
+        await execFileAsync5("where", [cmd], { cwd });
+      } else {
+        await execFileAsync5("sh", ["-c", `command -v ${cmd}`], { cwd });
+      }
+    } catch {
+      missing.push(`command: ${cmd}`);
+    }
+  }
+  for (const mod of env.required_python_modules ?? []) {
+    log(`preflight: checking Python module "${mod}"`);
+    try {
+      await execFileAsync5("python3", ["-c", `import ${mod}`], { cwd });
+    } catch {
+      missing.push(`python module: ${mod}`);
+    }
+  }
+  if (missing.length > 0) {
+    throw new Error(
+      `Preflight checks failed \u2014 missing dependencies:
+${missing.map((m) => `  \u2022 ${m}`).join("\n")}
+
+Install the missing dependencies before running this eval.`
+    );
+  }
+}
 var import_node_child_process11, import_node_crypto11, import_node_fs16, import_promises35, import_node_path47, import_node_util7, import_micromatch2, execFileAsync3, WORKSPACE_GIT_TIMEOUT_MS;
 var init_orchestrator = __esm({
   "src/evaluation/orchestrator.ts"() {
@@ -22931,7 +23092,8 @@ function parseWorkspaceConfig(raw, evalFileDir) {
   const workspacePath = typeof obj.path === "string" ? obj.path : void 0;
   const mode = explicitMode ?? (workspacePath ? "static" : void 0);
   const docker = parseDockerWorkspaceConfig(obj.docker);
-  if (!template && !isolation && !repos && !hooks && !mode && !workspacePath && !docker)
+  const env = parseWorkspaceEnvConfig(obj.env);
+  if (!template && !isolation && !repos && !hooks && !mode && !workspacePath && !docker && !env)
     return void 0;
   return {
     ...template !== void 0 && { template },
@@ -22940,7 +23102,19 @@ function parseWorkspaceConfig(raw, evalFileDir) {
     ...hooks !== void 0 && { hooks },
     ...mode !== void 0 && { mode },
     ...workspacePath !== void 0 && { path: workspacePath },
-    ...docker !== void 0 && { docker }
+    ...docker !== void 0 && { docker },
+    ...env !== void 0 && { env }
+  };
+}
+function parseWorkspaceEnvConfig(raw) {
+  if (!isJsonObject(raw)) return void 0;
+  const obj = raw;
+  const required_commands = Array.isArray(obj.required_commands) ? obj.required_commands.filter((c) => typeof c === "string") : void 0;
+  const required_python_modules = Array.isArray(obj.required_python_modules) ? obj.required_python_modules.filter((m) => typeof m === "string") : void 0;
+  if (!required_commands?.length && !required_python_modules?.length) return void 0;
+  return {
+    ...required_commands?.length && { required_commands },
+    ...required_python_modules?.length && { required_python_modules }
   };
 }
 function parseDockerWorkspaceConfig(raw) {
@@ -24966,8 +25140,8 @@ init_cjs_shims();
 
 // src/evaluation/generators/rubric-generator.ts
 init_cjs_shims();
-var import_ai4 = require("ai");
 var import_zod6 = require("zod");
+init_types2();
 var rubricItemSchema = import_zod6.z.object({
   id: import_zod6.z.string().describe("Short identifier for this rubric (e.g., clarity, completeness)"),
   outcome: import_zod6.z.string().describe("Concrete expected outcome for this rubric item"),
@@ -24980,10 +25154,6 @@ var rubricGenerationSchema = import_zod6.z.object({
 async function generateRubrics(options) {
   const { criteria, question, referenceAnswer, provider } = options;
   const prompt = buildPrompt(criteria, question, referenceAnswer);
-  const model = provider.asLanguageModel?.();
-  if (!model) {
-    throw new Error("Provider does not support language model interface");
-  }
   const system = `You are an expert at creating evaluation rubrics.
 You must return a valid JSON object matching this schema:
 {
@@ -25000,11 +25170,11 @@ You must return a valid JSON object matching this schema:
   let lastError;
   for (let attempt = 1; attempt <= 3; attempt++) {
     try {
-      const { text } = await (0, import_ai4.generateText)({
-        model,
-        system,
-        prompt
+      const response = await provider.invoke({
+        question: prompt,
+        systemPrompt: system
       });
+      const text = extractLastAssistantContent2(response.output);
       const cleaned = text.replace(/```json\n?|```/g, "").trim();
       result = rubricGenerationSchema.parse(JSON.parse(cleaned));
       break;