@agentv/core 3.14.6 → 4.0.0

package/dist/index.cjs

@@ -1315,12 +1315,12 @@ function serializeAttributeValue(value) {
   if (Array.isArray(value)) return { arrayValue: { values: value.map(serializeAttributeValue) } };
   return { stringValue: String(value) };
 }
-var import_promises35, import_node_path51, OtlpJsonFileExporter;
+var import_promises35, import_node_path52, OtlpJsonFileExporter;
 var init_otlp_json_file_exporter = __esm({
   "src/observability/otlp-json-file-exporter.ts"() {
     "use strict";
     import_promises35 = require("fs/promises");
-    import_node_path51 = require("path");
+    import_node_path52 = require("path");
     OtlpJsonFileExporter = class {
       // biome-ignore lint/suspicious/noExplicitAny: serialized span data
       spans = [];
@@ -1359,7 +1359,7 @@ var init_otlp_json_file_exporter = __esm({
       }
       async flush() {
         if (this.spans.length === 0) return;
-        await (0, import_promises35.mkdir)((0, import_node_path51.dirname)(this.filePath), { recursive: true });
+        await (0, import_promises35.mkdir)((0, import_node_path52.dirname)(this.filePath), { recursive: true });
         const otlpJson = {
           resourceSpans: [
             {
@@ -1383,9 +1383,11 @@ var init_otlp_json_file_exporter = __esm({
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
+  COMMON_TARGET_SETTINGS: () => COMMON_TARGET_SETTINGS,
   CodeEvaluator: () => CodeEvaluator,
   CompositeEvaluator: () => CompositeEvaluator,
   CostEvaluator: () => CostEvaluator,
+  DEFAULT_CATEGORY: () => DEFAULT_CATEGORY,
   DEFAULT_EVALUATOR_TEMPLATE: () => DEFAULT_EVALUATOR_TEMPLATE,
   DEFAULT_EVAL_PATTERNS: () => DEFAULT_EVAL_PATTERNS,
   DEFAULT_EXPLORATION_TOOLS: () => DEFAULT_EXPLORATION_TOOLS,
@@ -1439,6 +1441,7 @@ __export(index_exports, {
   createTempWorkspace: () => createTempWorkspace,
   deepEqual: () => deepEqual,
   defineConfig: () => defineConfig,
+  deriveCategory: () => deriveCategory,
   detectFormat: () => detectFormat,
   discoverAssertions: () => discoverAssertions,
   discoverCopilotSessions: () => discoverCopilotSessions,
@@ -1452,7 +1455,9 @@ __export(index_exports, {
   explorationRatio: () => explorationRatio,
   extractCacheConfig: () => extractCacheConfig,
   extractFailOnError: () => extractFailOnError,
+  extractImageBlocks: () => extractImageBlocks,
   extractJsonBlob: () => extractJsonBlob,
+  extractLastAssistantContent: () => extractLastAssistantContent,
   extractTargetFromSuite: () => extractTargetFromSuite,
   extractTargetsFromSuite: () => extractTargetsFromSuite,
   extractTargetsFromTestCase: () => extractTargetsFromTestCase,
@@ -1466,12 +1471,15 @@ __export(index_exports, {
   getAgentvHome: () => getAgentvHome,
   getOutputFilenames: () => getOutputFilenames,
   getSubagentsRoot: () => getSubagentsRoot,
+  getTextContent: () => getTextContent,
   getTraceStateRoot: () => getTraceStateRoot,
   getWorkspacePath: () => getWorkspacePath,
   getWorkspacePoolRoot: () => getWorkspacePoolRoot,
   getWorkspacesRoot: () => getWorkspacesRoot,
   initializeBaseline: () => initializeBaseline,
   isAgentSkillsFormat: () => isAgentSkillsFormat,
+  isContent: () => isContent,
+  isContentArray: () => isContentArray,
   isEvaluatorKind: () => isEvaluatorKind,
   isJsonObject: () => isJsonObject,
   isJsonValue: () => isJsonValue,
@@ -1533,6 +1541,29 @@ __export(index_exports, {
 });
 module.exports = __toCommonJS(index_exports);
 
+// src/evaluation/content.ts
+var CONTENT_TYPES = /* @__PURE__ */ new Set(["text", "image", "file"]);
+function isContent(value) {
+  if (!value || typeof value !== "object") return false;
+  const v = value;
+  return typeof v.type === "string" && CONTENT_TYPES.has(v.type);
+}
+function isContentArray(value) {
+  return Array.isArray(value) && value.length > 0 && value.every(isContent);
+}
+function getTextContent(content) {
+  if (content == null) return "";
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  const parts = [];
+  for (const block of content) {
+    if (block.type === "text") {
+      parts.push(block.text);
+    }
+  }
+  return parts.join("\n");
+}
+
 // src/evaluation/types.ts
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
 var TEST_MESSAGE_ROLES = TEST_MESSAGE_ROLE_VALUES;
@@ -2411,15 +2442,23 @@ var TEMPLATE_VARIABLES = {
   INPUT: "input",
   OUTPUT: "output",
   FILE_CHANGES: "file_changes",
+  /** @deprecated Use INPUT instead — resolves to the same text value. */
   INPUT_TEXT: "input_text",
+  /** @deprecated Use OUTPUT instead — resolves to the same text value. */
   OUTPUT_TEXT: "output_text",
+  /** @deprecated Use EXPECTED_OUTPUT instead — resolves to the same text value. */
   EXPECTED_OUTPUT_TEXT: "expected_output_text"
 };
 var VALID_TEMPLATE_VARIABLES = new Set(Object.values(TEMPLATE_VARIABLES));
 var REQUIRED_TEMPLATE_VARIABLES = /* @__PURE__ */ new Set([
-  TEMPLATE_VARIABLES.OUTPUT_TEXT,
+  TEMPLATE_VARIABLES.OUTPUT,
   TEMPLATE_VARIABLES.EXPECTED_OUTPUT
 ]);
+var DEPRECATED_TEMPLATE_VARIABLES = /* @__PURE__ */ new Map([
+  [TEMPLATE_VARIABLES.INPUT_TEXT, TEMPLATE_VARIABLES.INPUT],
+  [TEMPLATE_VARIABLES.OUTPUT_TEXT, TEMPLATE_VARIABLES.OUTPUT],
+  [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT, TEMPLATE_VARIABLES.EXPECTED_OUTPUT]
+]);
 
 // src/evaluation/validation/prompt-validator.ts
 var ANSI_YELLOW3 = "\x1B[33m";
@@ -2441,16 +2480,29 @@ function validateTemplateVariables(content, source) {
     }
     match = variablePattern.exec(content);
   }
-  const hasCandidateAnswer = foundVariables.has(TEMPLATE_VARIABLES.OUTPUT_TEXT);
+  const hasCandidateAnswer = foundVariables.has(TEMPLATE_VARIABLES.OUTPUT) || foundVariables.has(TEMPLATE_VARIABLES.OUTPUT_TEXT);
   const hasExpectedOutput = foundVariables.has(TEMPLATE_VARIABLES.EXPECTED_OUTPUT);
   const hasRequiredFields = hasCandidateAnswer || hasExpectedOutput;
   if (!hasRequiredFields) {
     throw new Error(
       `Missing required fields. Must include at least one of:
-  - {{ ${TEMPLATE_VARIABLES.OUTPUT_TEXT} }}
+  - {{ ${TEMPLATE_VARIABLES.OUTPUT} }}
   - {{ ${TEMPLATE_VARIABLES.EXPECTED_OUTPUT} }}`
     );
   }
+  const deprecatedUsed = [];
+  for (const [deprecated, replacement] of DEPRECATED_TEMPLATE_VARIABLES) {
+    if (foundVariables.has(deprecated)) {
+      deprecatedUsed.push(`{{ ${deprecated} }} \u2192 {{ ${replacement} }}`);
+    }
+  }
+  if (deprecatedUsed.length > 0) {
+    console.warn(
+      `${ANSI_YELLOW3}Warning: Template at ${source} uses deprecated variable names:
+  ${deprecatedUsed.join("\n  ")}
+  These still work but will be removed in a future version.${ANSI_RESET4}`
+    );
+  }
   if (invalidVariables.length > 0) {
     const warningMessage = `${ANSI_YELLOW3}Warning: Custom evaluator template at ${source}
   Contains invalid variables: ${invalidVariables.map((v) => `{{ ${v} }}`).join(", ")}
@@ -3868,6 +3920,19 @@ function asString2(value) {
 }
 
 // src/evaluation/loaders/message-processor.ts
+var IMAGE_MEDIA_TYPES = {
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".gif": "image/gif",
+  ".webp": "image/webp",
+  ".svg": "image/svg+xml",
+  ".bmp": "image/bmp"
+};
+function detectImageMediaType(filePath) {
+  const ext = import_node_path6.default.extname(filePath).toLowerCase();
+  return IMAGE_MEDIA_TYPES[ext];
+}
 var ANSI_YELLOW5 = "\x1B[33m";
 var ANSI_RESET6 = "\x1B[0m";
 async function processMessages(options) {
@@ -3933,6 +3998,47 @@ async function processMessages(options) {
         }
         continue;
       }
+      if (segmentType === "image") {
+        const rawValue = asString3(rawSegment.value);
+        if (!rawValue) {
+          continue;
+        }
+        const { displayPath, resolvedPath, attempted } = await resolveFileReference2(
+          rawValue,
+          searchRoots
+        );
+        if (!resolvedPath) {
+          const attempts = attempted.length ? ["  Tried:", ...attempted.map((candidate) => `    ${candidate}`)] : void 0;
+          const context2 = messageType === "input" ? "" : " in expected_output";
+          logWarning3(`Image file not found${context2}: ${displayPath}`, attempts);
+          continue;
+        }
+        const mediaType = detectImageMediaType(resolvedPath);
+        if (!mediaType) {
+          logWarning3(
+            `Unsupported image extension for ${displayPath}. Supported: ${Object.keys(IMAGE_MEDIA_TYPES).join(", ")}`
+          );
+          continue;
+        }
+        try {
+          const imageBuffer = await (0, import_promises6.readFile)(resolvedPath);
+          const base64 = imageBuffer.toString("base64");
+          processedContent.push({
+            type: "image",
+            media_type: mediaType,
+            source: `data:${mediaType};base64,${base64}`
+          });
+          if (verbose) {
+            const label = messageType === "input" ? "[Image]" : "[Expected Output Image]";
+            console.log(`  ${label} Found: ${displayPath}`);
+            console.log(`    Resolved to: ${resolvedPath} (${mediaType})`);
+          }
+        } catch (error) {
+          const context2 = messageType === "input" ? "" : " expected output";
+          logWarning3(`Could not read${context2} image ${resolvedPath}: ${error.message}`);
+        }
+        continue;
+      }
       const clonedSegment = cloneJsonObject(rawSegment);
       processedContent.push(clonedSegment);
       const inlineValue = clonedSegment.value;
@@ -4010,6 +4116,46 @@ async function processExpectedMessages(options) {
           }
           continue;
         }
+        if (segmentType === "image") {
+          const rawValue = asString3(rawSegment.value);
+          if (!rawValue) {
+            continue;
+          }
+          const { displayPath, resolvedPath, attempted } = await resolveFileReference2(
+            rawValue,
+            searchRoots
+          );
+          if (!resolvedPath) {
+            const attempts = attempted.length ? ["  Tried:", ...attempted.map((candidate) => `    ${candidate}`)] : void 0;
+            logWarning3(`Image file not found in expected_output: ${displayPath}`, attempts);
+            continue;
+          }
+          const mediaType = detectImageMediaType(resolvedPath);
+          if (!mediaType) {
+            logWarning3(
+              `Unsupported image extension for ${displayPath}. Supported: ${Object.keys(IMAGE_MEDIA_TYPES).join(", ")}`
+            );
+            continue;
+          }
+          try {
+            const imageBuffer = await (0, import_promises6.readFile)(resolvedPath);
+            const base64 = imageBuffer.toString("base64");
+            processedContent.push({
+              type: "image",
+              media_type: mediaType,
+              source: `data:${mediaType};base64,${base64}`
+            });
+            if (verbose) {
+              console.log(`  [Expected Output Image] Found: ${displayPath}`);
+              console.log(`    Resolved to: ${resolvedPath} (${mediaType})`);
+            }
+          } catch (error) {
+            logWarning3(
+              `Could not read expected output image ${resolvedPath}: ${error.message}`
+            );
+          }
+          continue;
+        }
         processedContent.push(cloneJsonObject(rawSegment));
       }
       segment.content = processedContent;
@@ -4256,7 +4402,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
     const userFilePaths = collectResolvedInputFilePaths(inputMessages);
     const testCase = {
       id,
-      eval_set: evalSetName,
+      dataset: evalSetName,
       conversation_id: conversationId,
       question,
       input: inputMessages,
@@ -4527,7 +4673,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
   }
   const suite = interpolated;
   const evalSetNameFromSuite = asString5(suite.name)?.trim();
-  const fallbackEvalSet = import_node_path8.default.basename(absoluteTestPath).replace(/\.ya?ml$/i, "") || "eval";
+  const fallbackEvalSet = import_node_path8.default.basename(absoluteTestPath).replace(/\.eval\.ya?ml$/i, "").replace(/\.ya?ml$/i, "") || "eval";
   const evalSetName = evalSetNameFromSuite && evalSetNameFromSuite.length > 0 ? evalSetNameFromSuite : fallbackEvalSet;
   const rawTestcases = resolveTests(suite);
   const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm-grader";
@@ -4648,7 +4794,8 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
     const caseTargets = extractTargetsFromTestCase(evalcase);
     const testCase = {
       id,
-      eval_set: evalSetName,
+      dataset: evalSetName,
+      category: options?.category,
       conversation_id: conversationId,
       question,
       input: inputMessages,
@@ -5690,6 +5837,48 @@ var import_node_fs4 = require("fs");
 var import_promises10 = require("fs/promises");
 var import_node_path12 = __toESM(require("path"), 1);
 
+// src/evaluation/providers/claude-content.ts
+function toContentArray(content) {
+  if (!Array.isArray(content)) return void 0;
+  let hasNonText = false;
+  const blocks = [];
+  for (const part of content) {
+    if (!part || typeof part !== "object") continue;
+    const p = part;
+    if (p.type === "text" && typeof p.text === "string") {
+      blocks.push({ type: "text", text: p.text });
+    } else if (p.type === "image" && typeof p.source === "object" && p.source !== null) {
+      const src = p.source;
+      const mediaType = typeof p.media_type === "string" ? p.media_type : typeof src.media_type === "string" ? src.media_type : "application/octet-stream";
+      const data = typeof src.data === "string" ? `data:${mediaType};base64,${src.data}` : typeof p.url === "string" ? p.url : "";
+      blocks.push({ type: "image", media_type: mediaType, source: data });
+      hasNonText = true;
+    } else if (p.type === "tool_use") {
+    } else if (p.type === "tool_result") {
+    }
+  }
+  return hasNonText && blocks.length > 0 ? blocks : void 0;
+}
+function extractTextContent(content) {
+  if (typeof content === "string") {
+    return content;
+  }
+  if (!Array.isArray(content)) {
+    return void 0;
+  }
+  const textParts = [];
+  for (const part of content) {
+    if (!part || typeof part !== "object") {
+      continue;
+    }
+    const p = part;
+    if (p.type === "text" && typeof p.text === "string") {
+      textParts.push(p.text);
+    }
+  }
+  return textParts.length > 0 ? textParts.join("\n") : void 0;
+}
+
 // src/evaluation/providers/claude-log-tracker.ts
 var GLOBAL_LOGS_KEY = Symbol.for("agentv.claudeLogs");
 var GLOBAL_SUBSCRIBERS_KEY = Symbol.for("agentv.claudeLogSubscribers");
@@ -5855,11 +6044,12 @@ var ClaudeCliProvider = class {
             if (betaMessage && typeof betaMessage === "object") {
               const msg = betaMessage;
               const content = msg.content;
+              const structuredContent = toContentArray(content);
               const textContent = extractTextContent(content);
               const toolCalls = extractToolCalls(content);
               const outputMsg = {
                 role: "assistant",
-                content: textContent,
+                content: structuredContent ?? textContent,
                 toolCalls: toolCalls.length > 0 ? toolCalls : void 0
               };
               output.push(outputMsg);
@@ -6198,25 +6388,6 @@ function summarizeEvent(event) {
       return void 0;
   }
 }
-function extractTextContent(content) {
-  if (typeof content === "string") {
-    return content;
-  }
-  if (!Array.isArray(content)) {
-    return void 0;
-  }
-  const textParts = [];
-  for (const part of content) {
-    if (!part || typeof part !== "object") {
-      continue;
-    }
-    const p = part;
-    if (p.type === "text" && typeof p.text === "string") {
-      textParts.push(p.text);
-    }
-  }
-  return textParts.length > 0 ? textParts.join("\n") : void 0;
-}
 function extractToolCalls(content) {
   if (!Array.isArray(content)) {
     return [];
@@ -6389,11 +6560,12 @@ var ClaudeSdkProvider = class {
             if (betaMessage && typeof betaMessage === "object") {
               const msg = betaMessage;
               const content = msg.content;
-              const textContent = extractTextContent2(content);
+              const structuredContent = toContentArray(content);
+              const textContent = extractTextContent(content);
               const toolCalls = extractToolCalls2(content);
               const outputMsg = {
                 role: "assistant",
-                content: textContent,
+                content: structuredContent ?? textContent,
                 toolCalls: toolCalls.length > 0 ? toolCalls : void 0
               };
               output.push(outputMsg);
@@ -6511,25 +6683,6 @@ var ClaudeSdkProvider = class {
     }
   }
 };
-function extractTextContent2(content) {
-  if (typeof content === "string") {
-    return content;
-  }
-  if (!Array.isArray(content)) {
-    return void 0;
-  }
-  const textParts = [];
-  for (const part of content) {
-    if (!part || typeof part !== "object") {
-      continue;
-    }
-    const p = part;
-    if (p.type === "text" && typeof p.text === "string") {
-      textParts.push(p.text);
-    }
-  }
-  return textParts.length > 0 ? textParts.join("\n") : void 0;
-}
 function extractToolCalls2(content) {
   if (!Array.isArray(content)) {
     return [];
@@ -6753,7 +6906,7 @@ function convertMessages(messages) {
   return messages.map((msg) => ({
     role: msg.role,
     name: msg.name,
-    content: msg.content,
+    content: isContentArray(msg.content) ? msg.content : typeof msg.content === "string" ? msg.content : void 0,
     toolCalls: msg.tool_calls?.map((tc) => ({
       tool: tc.tool,
       input: tc.input,
@@ -9007,6 +9160,35 @@ function extractPiTextContent(content) {
   }
   return textParts.length > 0 ? textParts.join("\n") : void 0;
 }
+function toPiContentArray(content) {
+  if (!Array.isArray(content)) return void 0;
+  let hasNonText = false;
+  const blocks = [];
+  for (const part of content) {
+    if (!part || typeof part !== "object") continue;
+    const p = part;
+    if (p.type === "text" && typeof p.text === "string") {
+      blocks.push({ type: "text", text: p.text });
+    } else if (p.type === "image") {
+      const mediaType = typeof p.media_type === "string" ? p.media_type : "application/octet-stream";
+      let source = "";
+      if (typeof p.source === "object" && p.source !== null) {
+        const src = p.source;
+        const srcMediaType = typeof src.media_type === "string" ? src.media_type : mediaType;
+        source = typeof src.data === "string" ? `data:${srcMediaType};base64,${src.data}` : "";
+      }
+      if (!source && typeof p.url === "string") {
+        source = p.url;
+      }
+      if (source) {
+        blocks.push({ type: "image", media_type: mediaType, source });
+        hasNonText = true;
+      }
+    } else if (p.type === "tool_use" || p.type === "tool_result") {
+    }
+  }
+  return hasNonText && blocks.length > 0 ? blocks : void 0;
+}
 function toFiniteNumber(value) {
   if (typeof value === "number" && Number.isFinite(value)) return value;
   return void 0;
@@ -10178,7 +10360,8 @@ function convertAgentMessage(message, toolTrackers, completedToolResults) {
   }
   const msg = message;
   const role = typeof msg.role === "string" ? msg.role : "unknown";
-  const content = extractPiTextContent(msg.content);
+  const structuredContent = toPiContentArray(msg.content);
+  const content = structuredContent ?? extractPiTextContent(msg.content);
   const toolCalls = extractToolCalls4(msg.content, toolTrackers, completedToolResults);
   const startTimeVal = typeof msg.timestamp === "number" ? new Date(msg.timestamp).toISOString() : typeof msg.timestamp === "string" ? msg.timestamp : void 0;
   let msgTokenUsage;
@@ -10440,6 +10623,12 @@ var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set([
   "FILES",
   "OUTPUT_FILE"
 ]);
+var COMMON_TARGET_SETTINGS = [
+  "provider_batching",
+  "providerBatching",
+  "subagent_mode_allowed",
+  "subagentModeAllowed"
+];
 var BASE_TARGET_SCHEMA = import_zod3.z.object({
   name: import_zod3.z.string().min(1, "target name is required"),
   provider: import_zod3.z.string().min(1, "provider is required"),
@@ -10448,7 +10637,8 @@ var BASE_TARGET_SCHEMA = import_zod3.z.object({
   // backward compat
   workers: import_zod3.z.number().int().min(1).optional(),
   workspace_template: import_zod3.z.string().optional(),
-  workspaceTemplate: import_zod3.z.string().optional()
+  workspaceTemplate: import_zod3.z.string().optional(),
+  subagent_mode_allowed: import_zod3.z.boolean().optional()
 }).passthrough();
 var DEFAULT_AZURE_API_VERSION = "2024-12-01-preview";
 var DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
@@ -10511,42 +10701,40 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
   const providerBatching = resolveOptionalBoolean(
     parsed.provider_batching ?? parsed.providerBatching
   );
+  const subagentModeAllowed = resolveOptionalBoolean(
+    parsed.subagent_mode_allowed ?? parsed.subagentModeAllowed
+  );
+  const base = {
+    name: parsed.name,
+    graderTarget: parsed.grader_target ?? parsed.judge_target,
+    workers: parsed.workers,
+    providerBatching,
+    subagentModeAllowed
+  };
   switch (provider) {
     case "openai":
       return {
         kind: "openai",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveOpenAIConfig(parsed, env)
       };
     case "openrouter":
       return {
         kind: "openrouter",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveOpenRouterConfig(parsed, env)
       };
     case "azure":
     case "azure-openai":
       return {
         kind: "azure",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveAzureConfig(parsed, env)
       };
     case "anthropic":
       return {
         kind: "anthropic",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveAnthropicConfig(parsed, env)
       };
     case "gemini":
@@ -10554,68 +10742,47 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
     case "google-gemini":
       return {
         kind: "gemini",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveGeminiConfig(parsed, env)
       };
     case "codex":
     case "codex-cli":
       return {
         kind: "codex",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveCodexConfig(parsed, env, evalFilePath)
       };
     case "copilot-sdk":
     case "copilot_sdk":
       return {
         kind: "copilot-sdk",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveCopilotSdkConfig(parsed, env, evalFilePath)
       };
     case "copilot":
     case "copilot-cli":
       return {
         kind: "copilot-cli",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveCopilotCliConfig(parsed, env, evalFilePath)
       };
     case "copilot-log":
       return {
         kind: "copilot-log",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveCopilotLogConfig(parsed, env)
       };
     case "pi":
     case "pi-coding-agent":
       return {
         kind: "pi-coding-agent",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolvePiCodingAgentConfig(parsed, env, evalFilePath)
       };
     case "pi-cli":
       return {
         kind: "pi-cli",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolvePiCliConfig(parsed, env, evalFilePath)
       };
     case "claude":
@@ -10623,38 +10790,26 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
     case "claude-cli":
       return {
         kind: "claude-cli",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveClaudeConfig(parsed, env, evalFilePath)
       };
     case "claude-sdk":
       return {
         kind: "claude-sdk",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveClaudeConfig(parsed, env, evalFilePath)
       };
     case "mock":
       return {
         kind: "mock",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveMockConfig(parsed)
       };
     case "vscode":
     case "vscode-insiders":
       return {
         kind: provider,
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders", evalFilePath)
       };
     case "agentv": {
@@ -10667,29 +10822,21 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
       const temperature = typeof parsed.temperature === "number" ? parsed.temperature : 0;
       return {
         kind: "agentv",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
+        ...base,
         workers: typeof parsed.workers === "number" ? parsed.workers : void 0,
-        providerBatching,
         config: { model, temperature }
       };
     }
     case "cli":
       return {
         kind: "cli",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveCliConfig(parsed, env, evalFilePath)
       };
     default:
       return {
         kind: "cli",
-        name: parsed.name,
-        graderTarget: parsed.grader_target ?? parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
+        ...base,
         config: resolveDiscoveredProviderConfig(parsed, provider, env, evalFilePath)
       };
   }
@@ -11317,8 +11464,8 @@ function resolveCliConfig(target, env, evalFilePath) {
   const parseResult = CliTargetInputSchema.safeParse(target, { errorMap: cliErrorMap });
   if (!parseResult.success) {
     const firstError = parseResult.error.errors[0];
-    const path50 = firstError?.path.join(".") || "";
-    const prefix = path50 ? `${target.name} ${path50}: ` : `${target.name}: `;
+    const path51 = firstError?.path.join(".") || "";
+    const prefix = path51 ? `${target.name} ${path51}: ` : `${target.name}: `;
     throw new Error(`${prefix}${firstError?.message}`);
   }
   const normalized = normalizeCliTargetInput(parseResult.data, env, evalFilePath);
@@ -13007,6 +13154,41 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
   }
 }
 
+// src/evaluation/providers/types.ts
+var AGENT_PROVIDER_KINDS = [
+  "codex",
+  "copilot-sdk",
+  "copilot-cli",
+  "pi-coding-agent",
+  "pi-cli",
+  "claude",
+  "claude-cli",
+  "claude-sdk",
+  "vscode",
+  "vscode-insiders"
+];
+function extractLastAssistantContent(messages) {
+  if (!messages || messages.length === 0) {
+    return "";
+  }
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msg = messages[i];
+    if (msg.role === "assistant" && msg.content !== void 0) {
+      if (typeof msg.content === "string") {
+        return msg.content;
+      }
+      if (isContentArray(msg.content)) {
+        return getTextContent(msg.content);
+      }
+      return JSON.stringify(msg.content);
+    }
+  }
+  return "";
+}
+function isAgentProvider(provider) {
+  return provider ? AGENT_PROVIDER_KINDS.includes(provider.kind) : false;
+}
+
 // src/evaluation/providers/targets-file.ts
 var import_node_fs11 = require("fs");
 var import_promises27 = require("fs/promises");
@@ -13319,13 +13501,13 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
 async function execShellWithStdin(command, stdinPayload, options = {}) {
   const { mkdir: mkdir17, readFile: readFile16, rm: rm6, writeFile: writeFile9 } = await import("fs/promises");
   const { tmpdir: tmpdir3 } = await import("os");
-  const path50 = await import("path");
+  const path51 = await import("path");
   const { randomUUID: randomUUID10 } = await import("crypto");
-  const dir = path50.join(tmpdir3(), `agentv-exec-${randomUUID10()}`);
+  const dir = path51.join(tmpdir3(), `agentv-exec-${randomUUID10()}`);
   await mkdir17(dir, { recursive: true });
-  const stdinPath = path50.join(dir, "stdin.txt");
-  const stdoutPath = path50.join(dir, "stdout.txt");
-  const stderrPath = path50.join(dir, "stderr.txt");
+  const stdinPath = path51.join(dir, "stdin.txt");
+  const stdoutPath = path51.join(dir, "stdout.txt");
+  const stderrPath = path51.join(dir, "stderr.txt");
   await writeFile9(stdinPath, stdinPayload, "utf8");
   const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
   const { spawn: spawn5 } = await import("child_process");
@@ -13457,7 +13639,7 @@ async function createTargetProxy(options) {
         totalOutputTokens += response.tokenUsage.output;
       }
       const output = response.output ?? [];
-      const rawText = extractLastAssistantContent(output);
+      const rawText = extractLastAssistantContent2(output);
       const result = {
         output,
         rawText,
@@ -13515,7 +13697,7 @@ async function createTargetProxy(options) {
           const output = response.output ?? [];
           responses.push({
             output,
-            rawText: extractLastAssistantContent(output),
+            rawText: extractLastAssistantContent2(output),
             tokenUsage: response.tokenUsage
           });
         } catch (error) {
@@ -13572,7 +13754,7 @@ function readBody(req) {
     req.on("error", reject);
   });
 }
-function extractLastAssistantContent(messages) {
+function extractLastAssistantContent2(messages) {
   for (let i = messages.length - 1; i >= 0; i--) {
     const msg = messages[i];
     if (msg.role === "assistant" && msg.content !== void 0) {
@@ -13641,6 +13823,56 @@ function toCamelCaseDeep(obj) {
 
 // src/evaluation/evaluators/code-evaluator.ts
 var FILE_BACKED_OUTPUT_THRESHOLD = 5e4;
+var DATA_URI_RE = /^data:([^;]+);base64,(.+)$/s;
+async function materializeContentForGrader(messages, getWorkDir) {
+  if (!messages || messages.length === 0) return messages ?? null;
+  let hasAnyImage = false;
+  for (const msg of messages) {
+    if (isContentArray(msg.content)) {
+      for (const block of msg.content) {
+        if (block.type === "image") {
+          hasAnyImage = true;
+          break;
+        }
+      }
+    }
+    if (hasAnyImage) break;
+  }
+  if (!hasAnyImage) return messages;
+  let counter = 0;
+  const result = [];
+  for (const msg of messages) {
+    if (!isContentArray(msg.content)) {
+      result.push(msg);
+      continue;
+    }
+    if (!msg.content.some((b) => b.type === "image")) {
+      result.push(msg);
+      continue;
+    }
+    const blocks = [];
+    for (const block of msg.content) {
+      if (block.type !== "image") {
+        blocks.push({ ...block });
+        continue;
+      }
+      const img = block;
+      const match = DATA_URI_RE.exec(img.source);
+      if (match) {
+        const [, mediaType, base64Data] = match;
+        const ext = mediaType.split("/")[1] === "jpeg" ? "jpg" : mediaType.split("/")[1] ?? "bin";
+        const dir = await getWorkDir();
+        const filePath = (0, import_node_path38.join)(dir, `img-${counter++}.${ext}`);
+        await (0, import_promises28.writeFile)(filePath, Buffer.from(base64Data, "base64"));
+        blocks.push({ type: "image", media_type: img.media_type, path: filePath });
+      } else {
+        blocks.push({ type: "image", media_type: img.media_type, path: img.source });
+      }
+    }
+    result.push({ ...msg, content: blocks });
+  }
+  return result;
+}
 var CodeEvaluator = class {
   kind = "code-grader";
   command;
@@ -13656,7 +13888,18 @@ var CodeEvaluator = class {
     this.target = options.target;
   }
   async evaluate(context2) {
-    let outputForPayload = context2.output ?? null;
+    let imageTmpDir;
+    const getImageDir = async () => {
+      if (!imageTmpDir) {
+        imageTmpDir = await (0, import_promises28.mkdtemp)((0, import_node_path38.join)((0, import_node_os7.tmpdir)(), "agentv-img-"));
+      }
+      return imageTmpDir;
+    };
+    const materializedOutput = await materializeContentForGrader(
+      context2.output,
+      getImageDir
+    );
+    let outputForPayload = materializedOutput;
     let outputPath;
     if (outputForPayload) {
       const serialized = JSON.stringify(outputForPayload);
@@ -13669,12 +13912,17 @@ var CodeEvaluator = class {
     }
     const payload = {
       criteria: context2.evalCase.criteria,
-      expectedOutput: context2.evalCase.expected_output,
-      outputText: context2.candidate,
+      expectedOutput: await materializeContentForGrader(
+        context2.evalCase.expected_output,
+        getImageDir
+      ),
       output: outputForPayload,
       outputPath,
       inputFiles: context2.evalCase.file_paths,
-      input: context2.evalCase.input,
+      input: await materializeContentForGrader(
+        context2.evalCase.input,
+        getImageDir
+      ),
       trace: context2.trace ?? null,
       tokenUsage: context2.tokenUsage ?? null,
       costUsd: context2.costUsd ?? null,
@@ -13683,9 +13931,7 @@ var CodeEvaluator = class {
       endTime: context2.endTime ?? null,
       fileChanges: context2.fileChanges ?? null,
       workspacePath: context2.workspacePath ?? null,
-      config: this.config ?? null,
-      inputText: context2.evalCase.question,
-      expectedOutputText: context2.evalCase.reference_answer ?? ""
+      config: this.config ?? null
     };
     const inputPayload = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
     let proxyEnv;
@@ -13775,6 +14021,10 @@ var CodeEvaluator = class {
         await (0, import_promises28.rm)((0, import_node_path38.dirname)(outputPath), { recursive: true, force: true }).catch(() => {
         });
       }
+      if (imageTmpDir) {
+        await (0, import_promises28.rm)(imageTmpDir, { recursive: true, force: true }).catch(() => {
+        });
+      }
     }
   }
 };
@@ -13802,38 +14052,6 @@ ${tail}`;
 // src/evaluation/evaluators/composite.ts
 var import_ai3 = require("ai");
 
-// src/evaluation/providers/types.ts
-var AGENT_PROVIDER_KINDS = [
-  "codex",
-  "copilot-sdk",
-  "copilot-cli",
-  "pi-coding-agent",
-  "pi-cli",
-  "claude",
-  "claude-cli",
-  "claude-sdk",
-  "vscode",
-  "vscode-insiders"
-];
-function extractLastAssistantContent2(messages) {
-  if (!messages || messages.length === 0) {
-    return "";
-  }
-  for (let i = messages.length - 1; i >= 0; i--) {
-    const msg = messages[i];
-    if (msg.role === "assistant" && msg.content !== void 0) {
-      if (typeof msg.content === "string") {
-        return msg.content;
-      }
-      return JSON.stringify(msg.content);
-    }
-  }
-  return "";
-}
-function isAgentProvider(provider) {
-  return provider ? AGENT_PROVIDER_KINDS.includes(provider.kind) : false;
-}
-
 // src/evaluation/evaluators/llm-grader.ts
 var import_promises29 = __toESM(require("fs/promises"), 1);
 var import_node_path39 = __toESM(require("path"), 1);
@@ -13884,13 +14102,13 @@ Be concise and focused in your evaluation. Provide succinct, specific feedback r
 {{${TEMPLATE_VARIABLES.CRITERIA}}}
 
 [[ ## question ## ]]
-{{${TEMPLATE_VARIABLES.INPUT_TEXT}}}
+{{${TEMPLATE_VARIABLES.INPUT}}}
 
 [[ ## reference_answer ## ]]
-{{${TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT}}}
+{{${TEMPLATE_VARIABLES.EXPECTED_OUTPUT}}}
 
 [[ ## answer ## ]]
-{{${TEMPLATE_VARIABLES.OUTPUT_TEXT}}}`;
+{{${TEMPLATE_VARIABLES.OUTPUT}}}`;
 var freeformEvaluationSchema = import_zod4.z.object({
   score: import_zod4.z.number().min(0).max(1).describe("Score between 0.0 and 1.0"),
   assertions: import_zod4.z.array(
@@ -13962,21 +14180,19 @@ var LlmGraderEvaluator = class {
   async evaluateFreeform(context2, graderProvider) {
     const formattedQuestion = context2.promptInputs.question && context2.promptInputs.question.trim().length > 0 ? context2.promptInputs.question : context2.evalCase.question;
     const variables = {
-      [TEMPLATE_VARIABLES.INPUT]: JSON.stringify(context2.evalCase.input, null, 2),
-      [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: JSON.stringify(
-        context2.evalCase.expected_output,
-        null,
-        2
-      ),
-      [TEMPLATE_VARIABLES.OUTPUT]: JSON.stringify(context2.output ?? [], null, 2),
+      [TEMPLATE_VARIABLES.INPUT]: formattedQuestion.trim(),
+      [TEMPLATE_VARIABLES.OUTPUT]: context2.candidate.trim(),
+      [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context2.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.CRITERIA]: context2.evalCase.criteria.trim(),
       [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? "",
+      // Deprecated aliases — same values as the primary variables above
       [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
       [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context2.candidate.trim(),
       [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context2.evalCase.reference_answer ?? "").trim()
     };
     const systemPrompt = buildOutputSchema();
     const evaluatorTemplate = context2.evaluatorTemplateOverride ?? this.evaluatorTemplate ?? DEFAULT_EVALUATOR_TEMPLATE;
+    warnDeprecatedTemplateVars(evaluatorTemplate);
     let userPrompt = substituteVariables(evaluatorTemplate, variables);
     if (context2.fileChanges && !context2.evaluatorTemplateOverride && !this.evaluatorTemplate) {
       userPrompt += `
@@ -13988,13 +14204,15 @@ ${context2.fileChanges}`;
       userPrompt,
       systemPrompt
     };
+    const images = context2.output ? extractImageBlocks(context2.output) : [];
     try {
       const { data, tokenUsage } = await this.runWithRetry({
         context: context2,
         graderProvider,
         systemPrompt,
         userPrompt,
-        schema: freeformEvaluationSchema
+        schema: freeformEvaluationSchema,
+        images
       });
       const score = clampScore(data.score);
       const assertions = Array.isArray(data.assertions) ? data.assertions.slice(0, 8) : [];
@@ -14038,13 +14256,15 @@ ${context2.fileChanges}`;
       userPrompt: prompt,
       systemPrompt
     };
+    const images = context2.output ? extractImageBlocks(context2.output) : [];
     try {
       const { data, tokenUsage } = await this.runWithRetry({
         context: context2,
         graderProvider,
         systemPrompt,
         userPrompt: prompt,
-        schema: rubricEvaluationSchema
+        schema: rubricEvaluationSchema,
+        images
       });
       const { score, verdict, assertions } = calculateRubricScore(data, rubrics);
       return {
@@ -14081,13 +14301,15 @@ ${context2.fileChanges}`;
       userPrompt: prompt,
       systemPrompt
     };
+    const images = context2.output ? extractImageBlocks(context2.output) : [];
     try {
       const { data, tokenUsage } = await this.runWithRetry({
         context: context2,
         graderProvider,
         systemPrompt,
         userPrompt: prompt,
-        schema: scoreRangeEvaluationSchema
+        schema: scoreRangeEvaluationSchema,
+        images
       });
       const { score, verdict, assertions, details } = calculateScoreRangeResult(data, rubrics);
       return {
@@ -14217,7 +14439,7 @@ ${context2.fileChanges}`;
         evalCaseId: context2.evalCase.id,
         attempt: context2.attempt
       });
-      const assistantContent = extractLastAssistantContent2(response.output);
+      const assistantContent = extractLastAssistantContent(response.output);
       if (!assistantContent) {
         return {
           score: 0,
@@ -14294,12 +14516,17 @@ ${context2.fileChanges}`;
     const formattedQuestion = context2.promptInputs.question && context2.promptInputs.question.trim().length > 0 ? context2.promptInputs.question : context2.evalCase.question;
     const variables = {
       [TEMPLATE_VARIABLES.CRITERIA]: context2.evalCase.criteria.trim(),
+      [TEMPLATE_VARIABLES.INPUT]: formattedQuestion.trim(),
+      [TEMPLATE_VARIABLES.OUTPUT]: context2.candidate.trim(),
+      [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context2.evalCase.reference_answer ?? "").trim(),
+      [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? "",
+      // Deprecated aliases
       [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
       [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context2.candidate.trim(),
-      [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context2.evalCase.reference_answer ?? "").trim(),
-      [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? ""
+      [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context2.evalCase.reference_answer ?? "").trim()
     };
     if (this.evaluatorTemplate) {
+      warnDeprecatedTemplateVars(this.evaluatorTemplate);
       return substituteVariables(this.evaluatorTemplate, variables);
     }
     const config = context2.evaluator;
@@ -14350,11 +14577,16 @@ ${context2.fileChanges}`;
     if (this.evaluatorTemplate) {
       const variables = {
         [TEMPLATE_VARIABLES.CRITERIA]: context2.evalCase.criteria.trim(),
+        [TEMPLATE_VARIABLES.INPUT]: formattedQuestion.trim(),
+        [TEMPLATE_VARIABLES.OUTPUT]: context2.candidate.trim(),
+        [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context2.evalCase.reference_answer ?? "").trim(),
+        [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? "",
+        // Deprecated aliases
         [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
         [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context2.candidate.trim(),
-        [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context2.evalCase.reference_answer ?? "").trim(),
-        [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? ""
+        [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context2.evalCase.reference_answer ?? "").trim()
       };
+      warnDeprecatedTemplateVars(this.evaluatorTemplate);
       const customPrompt = substituteVariables(this.evaluatorTemplate, variables);
       const outputSchema = rubrics && rubrics.length > 0 ? buildRubricOutputSchema() : buildOutputSchema();
       return `${customPrompt}
@@ -14525,18 +14757,35 @@ ${outputSchema}`;
   // LLM mode retry logic
   // ---------------------------------------------------------------------------
   async runWithRetry(options) {
-    const { context: context2, graderProvider, systemPrompt, userPrompt, schema } = options;
+    const { context: context2, graderProvider, systemPrompt, userPrompt, schema, images } = options;
     let lastError;
     for (let attempt = 1; attempt <= 3; attempt++) {
       try {
         const model = graderProvider.asLanguageModel?.();
         if (model) {
-          const result = await (0, import_ai2.generateText)({
+          const modelOptions = {
+            ...this.maxOutputTokens ? { maxTokens: this.maxOutputTokens } : {},
+            ...typeof this.temperature === "number" ? { temperature: this.temperature } : {}
+          };
+          const hasImages = images && images.length > 0;
+          const result = hasImages ? await (0, import_ai2.generateText)({
+            model,
+            system: systemPrompt,
+            messages: [
+              {
+                role: "user",
+                content: [
+                  { type: "text", text: userPrompt },
+                  ...toAiSdkImageParts(images)
+                ]
+              }
+            ],
+            ...modelOptions
+          }) : await (0, import_ai2.generateText)({
             model,
             system: systemPrompt,
             prompt: userPrompt,
-            ...this.maxOutputTokens ? { maxTokens: this.maxOutputTokens } : {},
-            ...typeof this.temperature === "number" ? { temperature: this.temperature } : {}
+            ...modelOptions
           });
           const data2 = schema.parse(parseJsonFromText(result.text));
           const rawUsage = result.usage;
@@ -14551,7 +14800,7 @@ ${outputSchema}`;
           maxOutputTokens: this.maxOutputTokens,
           temperature: this.temperature
         });
-        const data = schema.parse(parseJsonFromText(extractLastAssistantContent2(response.output)));
+        const data = schema.parse(parseJsonFromText(extractLastAssistantContent(response.output)));
         return { data, providerResponse: response, tokenUsage: response.tokenUsage };
       } catch (e) {
         lastError = e instanceof Error ? e : new Error(String(e));
@@ -14596,6 +14845,26 @@ function substituteVariables(template, variables) {
     return variables[varName] ?? match;
   });
 }
+var ANSI_YELLOW8 = "\x1B[33m";
+var ANSI_RESET9 = "\x1B[0m";
+var warnedTemplateStrings = /* @__PURE__ */ new Set();
+function warnDeprecatedTemplateVars(template) {
+  if (warnedTemplateStrings.has(template)) return;
+  const used = [];
+  for (const [deprecated, replacement] of DEPRECATED_TEMPLATE_VARIABLES) {
+    if (new RegExp(`\\{\\{\\s*${deprecated}\\s*\\}\\}`).test(template)) {
+      used.push(`{{ ${deprecated} }} \u2192 {{ ${replacement} }}`);
+    }
+  }
+  if (used.length > 0) {
+    warnedTemplateStrings.add(template);
+    console.warn(
+      `${ANSI_YELLOW8}\u26A0 Deprecated template variables detected (they still work but will be removed in a future version):
+  ${used.join("\n  ")}
+  Update your custom evaluator template to use the new names.${ANSI_RESET9}`
+    );
+  }
+}
 function calculateRubricScore(result, rubrics) {
   const rubricMap = new Map(rubrics.map((rubric) => [rubric.id, rubric]));
   const assertions = [];
@@ -14690,6 +14959,26 @@ function calculateScoreRangeResult(result, rubrics) {
     }
   };
 }
+function extractImageBlocks(messages) {
+  const images = [];
+  for (const msg of messages) {
+    if (msg.role !== "assistant") continue;
+    if (!isContentArray(msg.content)) continue;
+    for (const block of msg.content) {
+      if (block.type === "image") {
+        images.push(block);
+      }
+    }
+  }
+  return images;
+}
+function toAiSdkImageParts(images) {
+  return images.map((img) => ({
+    type: "image",
+    image: img.source,
+    mediaType: img.media_type || void 0
+  }));
+}
 function resolveSandboxed(basePath, relativePath) {
   const resolved = import_node_path39.default.resolve(basePath, relativePath);
   if (!resolved.startsWith(basePath + import_node_path39.default.sep) && resolved !== basePath) {
@@ -15075,7 +15364,7 @@ var CompositeEvaluator = class {
         attempt: context2.attempt
       });
       const data = freeformEvaluationSchema.parse(
-        parseJsonFromText(extractLastAssistantContent2(response.output))
+        parseJsonFromText(extractLastAssistantContent(response.output))
       );
       const score = clampScore(data.score);
       const assertions = Array.isArray(data.assertions) ? data.assertions.slice(0, 8) : [];
@@ -15431,115 +15720,115 @@ var FieldAccuracyEvaluator = class {
    * Evaluate a single field against the expected value.
    */
   evaluateField(fieldConfig, candidateData, expectedData) {
-    const { path: path50, match, required = true, weight = 1 } = fieldConfig;
-    const candidateValue = resolvePath(candidateData, path50);
-    const expectedValue = resolvePath(expectedData, path50);
+    const { path: path51, match, required = true, weight = 1 } = fieldConfig;
+    const candidateValue = resolvePath(candidateData, path51);
+    const expectedValue = resolvePath(expectedData, path51);
     if (expectedValue === void 0) {
       return {
-        path: path50,
+        path: path51,
         score: 1,
         // No expected value means no comparison needed
         weight,
         hit: true,
-        message: `${path50}: no expected value`
+        message: `${path51}: no expected value`
       };
     }
     if (candidateValue === void 0) {
       if (required) {
         return {
-          path: path50,
+          path: path51,
           score: 0,
           weight,
           hit: false,
-          message: `${path50} (required, missing)`
+          message: `${path51} (required, missing)`
         };
       }
       return {
-        path: path50,
+        path: path51,
         score: 1,
         // Don't penalize missing optional fields
         weight: 0,
         // Zero weight means it won't affect the score
         hit: true,
-        message: `${path50}: optional field missing`
+        message: `${path51}: optional field missing`
       };
     }
     switch (match) {
       case "exact":
-        return this.compareExact(path50, candidateValue, expectedValue, weight);
+        return this.compareExact(path51, candidateValue, expectedValue, weight);
       case "numeric_tolerance":
         return this.compareNumericTolerance(
-          path50,
+          path51,
           candidateValue,
           expectedValue,
           fieldConfig,
           weight
         );
       case "date":
-        return this.compareDate(path50, candidateValue, expectedValue, fieldConfig, weight);
+        return this.compareDate(path51, candidateValue, expectedValue, fieldConfig, weight);
       default:
         return {
-          path: path50,
+          path: path51,
           score: 0,
           weight,
           hit: false,
-          message: `${path50}: unknown match type "${match}"`
+          message: `${path51}: unknown match type "${match}"`
         };
     }
   }
   /**
    * Exact equality comparison.
    */
-  compareExact(path50, candidateValue, expectedValue, weight) {
+  compareExact(path51, candidateValue, expectedValue, weight) {
     if (deepEqual(candidateValue, expectedValue)) {
       return {
-        path: path50,
+        path: path51,
         score: 1,
         weight,
         hit: true,
-        message: path50
+        message: path51
       };
     }
     if (typeof candidateValue !== typeof expectedValue) {
       return {
-        path: path50,
+        path: path51,
         score: 0,
         weight,
         hit: false,
-        message: `${path50} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
+        message: `${path51} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
       };
     }
     return {
-      path: path50,
+      path: path51,
       score: 0,
       weight,
       hit: false,
-      message: `${path50} (value mismatch)`
+      message: `${path51} (value mismatch)`
     };
   }
   /**
    * Numeric comparison with absolute or relative tolerance.
    */
-  compareNumericTolerance(path50, candidateValue, expectedValue, fieldConfig, weight) {
+  compareNumericTolerance(path51, candidateValue, expectedValue, fieldConfig, weight) {
     const { tolerance = 0, relative = false } = fieldConfig;
     const candidateNum = toNumber(candidateValue);
     const expectedNum = toNumber(expectedValue);
     if (candidateNum === null || expectedNum === null) {
       return {
-        path: path50,
+        path: path51,
         score: 0,
         weight,
         hit: false,
-        message: `${path50} (non-numeric value)`
+        message: `${path51} (non-numeric value)`
       };
     }
     if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
       return {
-        path: path50,
+        path: path51,
         score: 0,
         weight,
         hit: false,
-        message: `${path50} (invalid numeric value)`
+        message: `${path51} (invalid numeric value)`
       };
     }
     const diff = Math.abs(candidateNum - expectedNum);
@@ -15552,61 +15841,61 @@ var FieldAccuracyEvaluator = class {
     }
     if (withinTolerance) {
       return {
-        path: path50,
+        path: path51,
         score: 1,
         weight,
         hit: true,
-        message: `${path50} (within tolerance: diff=${diff.toFixed(2)})`
+        message: `${path51} (within tolerance: diff=${diff.toFixed(2)})`
       };
     }
     return {
-      path: path50,
+      path: path51,
       score: 0,
       weight,
       hit: false,
-      message: `${path50} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
+      message: `${path51} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
     };
   }
   /**
    * Date comparison with format normalization.
    */
-  compareDate(path50, candidateValue, expectedValue, fieldConfig, weight) {
+  compareDate(path51, candidateValue, expectedValue, fieldConfig, weight) {
     const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
     const candidateDate = parseDate(String(candidateValue), formats);
     const expectedDate = parseDate(String(expectedValue), formats);
     if (candidateDate === null) {
       return {
-        path: path50,
+        path: path51,
         score: 0,
         weight,
         hit: false,
-        message: `${path50} (unparseable candidate date)`
+        message: `${path51} (unparseable candidate date)`
       };
     }
     if (expectedDate === null) {
       return {
-        path: path50,
+        path: path51,
         score: 0,
         weight,
         hit: false,
-        message: `${path50} (unparseable expected date)`
+        message: `${path51} (unparseable expected date)`
       };
     }
     if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
       return {
-        path: path50,
+        path: path51,
         score: 1,
         weight,
         hit: true,
-        message: path50
+        message: path51
       };
     }
     return {
-      path: path50,
+      path: path51,
       score: 0,
       weight,
       hit: false,
-      message: `${path50} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
+      message: `${path51} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
     };
   }
   /**
@@ -15639,11 +15928,11 @@ var FieldAccuracyEvaluator = class {
     };
   }
 };
-function resolvePath(obj, path50) {
-  if (!path50 || !obj) {
+function resolvePath(obj, path51) {
+  if (!path51 || !obj) {
     return void 0;
   }
-  const parts = path50.split(/\.|\[|\]/).filter((p) => p.length > 0);
+  const parts = path51.split(/\.|\[|\]/).filter((p) => p.length > 0);
   let current = obj;
   for (const part of parts) {
     if (current === null || current === void 0) {
@@ -15935,11 +16224,12 @@ function assembleLlmGraderPrompt(input) {
 function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, evaluatorTemplateOverride) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const variables = {
-    [TEMPLATE_VARIABLES.INPUT]: JSON.stringify(evalCase.input, null, 2),
-    [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: JSON.stringify(evalCase.expected_output, null, 2),
-    [TEMPLATE_VARIABLES.OUTPUT]: JSON.stringify([], null, 2),
+    [TEMPLATE_VARIABLES.INPUT]: formattedQuestion.trim(),
+    [TEMPLATE_VARIABLES.OUTPUT]: candidate.trim(),
+    [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (evalCase.reference_answer ?? "").trim(),
     [TEMPLATE_VARIABLES.CRITERIA]: evalCase.criteria.trim(),
     [TEMPLATE_VARIABLES.FILE_CHANGES]: fileChanges ?? "",
+    // Deprecated aliases
     [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
     [TEMPLATE_VARIABLES.OUTPUT_TEXT]: candidate.trim(),
     [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (evalCase.reference_answer ?? "").trim()
@@ -16126,8 +16416,8 @@ var TokenUsageEvaluator = class {
 };
 
 // src/evaluation/evaluators/tool-trajectory.ts
-function getNestedValue(obj, path50) {
-  const parts = path50.split(".");
+function getNestedValue(obj, path51) {
+  const parts = path51.split(".");
   let current = obj;
   for (const part of parts) {
     if (current === null || current === void 0 || typeof current !== "object") {
@@ -16996,16 +17286,13 @@ async function executePromptTemplate(script, context2, config, timeoutMs) {
   const payload = {
     criteria: context2.evalCase.criteria,
     expectedOutput: context2.evalCase.expected_output,
-    outputText: context2.candidate,
     output: context2.output ?? null,
     inputFiles: context2.evalCase.file_paths,
     input: context2.evalCase.input,
     trace: context2.trace ?? null,
     fileChanges: context2.fileChanges ?? null,
     workspacePath: context2.workspacePath ?? null,
-    config: config ?? context2.config ?? null,
-    inputText: context2.evalCase.question,
-    expectedOutputText: context2.evalCase.reference_answer ?? ""
+    config: config ?? context2.config ?? null
   };
   const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
   const scriptPath = script[script.length - 1];
@@ -18685,7 +18972,8 @@ async function runEvaluation(options) {
           const budgetResult = {
             timestamp: (now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
             testId: evalCase.id,
-            eval_set: evalCase.eval_set,
+            dataset: evalCase.dataset,
+            category: evalCase.category,
             score: 0,
             assertions: [],
             output: [],
@@ -18721,7 +19009,8 @@ async function runEvaluation(options) {
           const haltResult = {
             timestamp: (now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
             testId: evalCase.id,
-            eval_set: evalCase.eval_set,
+            dataset: evalCase.dataset,
+            category: evalCase.category,
             score: 0,
             assertions: [],
             output: [],
@@ -19004,7 +19293,7 @@ async function runBatchEvaluation(options) {
     const tokenUsage = merged?.tokenUsage;
     const startTime = merged?.startTime;
     const endTime = merged?.endTime;
-    const candidate = extractLastAssistantContent2(output);
+    const candidate = extractLastAssistantContent(output);
     const providerError = extractProviderError(providerResponse);
     let result;
     try {
@@ -19412,7 +19701,7 @@ async function runEvalCase(options) {
   const tokenUsage = merged?.tokenUsage;
   const startTime = merged?.startTime;
   const endTime = merged?.endTime;
-  const candidate = extractLastAssistantContent2(output);
+  const candidate = extractLastAssistantContent(output);
   let fileChanges;
   if (baselineCommit && workspacePath) {
     try {
@@ -19720,7 +20009,8 @@ async function evaluateCandidate(options) {
   return {
     timestamp: completedAt.toISOString(),
     testId: evalCase.id,
-    eval_set: evalCase.eval_set,
+    dataset: evalCase.dataset,
+    category: evalCase.category,
     conversationId: evalCase.conversation_id,
     score: score.score,
     assertions: score.assertions,
@@ -20070,7 +20360,8 @@ function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs,
   return {
     timestamp: timestamp.toISOString(),
     testId: evalCase.id,
-    eval_set: evalCase.eval_set,
+    dataset: evalCase.dataset,
+    category: evalCase.category,
     conversationId: evalCase.conversation_id,
     score: 0,
     assertions: [{ text: `Error: ${message}`, passed: false }],
@@ -20643,6 +20934,18 @@ function trimBaselineResult(result) {
   return trimmed;
 }
 
+// src/evaluation/category.ts
+var import_node_path51 = __toESM(require("path"), 1);
+var DEFAULT_CATEGORY = "Uncategorized";
+function deriveCategory(relativePath) {
+  const parts = relativePath.split(import_node_path51.default.sep);
+  if (parts.length <= 1) {
+    return DEFAULT_CATEGORY;
+  }
+  const dirs = parts.slice(0, -1).filter((d) => d !== "evals");
+  return dirs.length > 0 ? dirs.join("/") : DEFAULT_CATEGORY;
+}
+
 // src/observability/otel-exporter.ts
 var OTEL_BACKEND_PRESETS = {
   langfuse: {
@@ -20766,7 +21069,7 @@ var OtelTraceExporter = class {
         rootSpan.setAttribute("gen_ai.system", "agentv");
         rootSpan.setAttribute("agentv.test_id", result.testId);
         rootSpan.setAttribute("agentv.target", result.target);
-        if (result.eval_set) rootSpan.setAttribute("agentv.eval_set", result.eval_set);
+        if (result.dataset) rootSpan.setAttribute("agentv.dataset", result.dataset);
         rootSpan.setAttribute("agentv.score", result.score);
         if (captureContent && result.output.length > 0) {
           const lastMsg = result.output[result.output.length - 1];
@@ -20975,7 +21278,7 @@ var OtelStreamingObserver = class {
     this.rootSpan.setAttribute("gen_ai.system", "agentv");
     this.rootSpan.setAttribute("agentv.test_id", testId);
     this.rootSpan.setAttribute("agentv.target", target);
-    if (evalSet) this.rootSpan.setAttribute("agentv.eval_set", evalSet);
+    if (evalSet) this.rootSpan.setAttribute("agentv.dataset", evalSet);
     this.rootCtx = this.api.trace.setSpan(this.api.context.active(), this.rootSpan);
   }
   /** Create and immediately export a tool span */
@@ -21151,9 +21454,11 @@ function createAgentKernel() {
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  COMMON_TARGET_SETTINGS,
   CodeEvaluator,
   CompositeEvaluator,
   CostEvaluator,
+  DEFAULT_CATEGORY,
   DEFAULT_EVALUATOR_TEMPLATE,
   DEFAULT_EVAL_PATTERNS,
   DEFAULT_EXPLORATION_TOOLS,
@@ -21207,6 +21512,7 @@ function createAgentKernel() {
   createTempWorkspace,
   deepEqual,
   defineConfig,
+  deriveCategory,
   detectFormat,
   discoverAssertions,
   discoverCopilotSessions,
@@ -21220,7 +21526,9 @@ function createAgentKernel() {
   explorationRatio,
   extractCacheConfig,
   extractFailOnError,
+  extractImageBlocks,
   extractJsonBlob,
+  extractLastAssistantContent,
   extractTargetFromSuite,
   extractTargetsFromSuite,
   extractTargetsFromTestCase,
@@ -21234,12 +21542,15 @@ function createAgentKernel() {
   getAgentvHome,
   getOutputFilenames,
   getSubagentsRoot,
+  getTextContent,
   getTraceStateRoot,
   getWorkspacePath,
   getWorkspacePoolRoot,
   getWorkspacesRoot,
   initializeBaseline,
   isAgentSkillsFormat,
+  isContent,
+  isContentArray,
   isEvaluatorKind,
   isJsonObject,
   isJsonValue,