@agentv/core 3.14.5 → 4.0.0

package/dist/index.js

@@ -1,4 +1,5 @@
 import {
+  COMMON_TARGET_SETTINGS,
   TEST_MESSAGE_ROLES,
   buildDirectoryChain,
   buildSearchRoots,
@@ -6,8 +7,11 @@ import {
   extractLastAssistantContent,
   fileExists,
   findGitRoot,
+  getTextContent,
   interpolateEnv,
   isAgentProvider,
+  isContent,
+  isContentArray,
   isEvaluatorKind,
   isJsonObject,
   isJsonValue,
@@ -19,7 +23,7 @@ import {
   readTextFile,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-HP5PFOVK.js";
+} from "./chunk-PXYYRDHH.js";
 import {
   AgentvProvider
 } from "./chunk-W5YDZWT4.js";
@@ -690,15 +694,23 @@ var TEMPLATE_VARIABLES = {
   INPUT: "input",
   OUTPUT: "output",
   FILE_CHANGES: "file_changes",
+  /** @deprecated Use INPUT instead — resolves to the same text value. */
   INPUT_TEXT: "input_text",
+  /** @deprecated Use OUTPUT instead — resolves to the same text value. */
   OUTPUT_TEXT: "output_text",
+  /** @deprecated Use EXPECTED_OUTPUT instead — resolves to the same text value. */
   EXPECTED_OUTPUT_TEXT: "expected_output_text"
 };
 var VALID_TEMPLATE_VARIABLES = new Set(Object.values(TEMPLATE_VARIABLES));
 var REQUIRED_TEMPLATE_VARIABLES = /* @__PURE__ */ new Set([
-  TEMPLATE_VARIABLES.OUTPUT_TEXT,
+  TEMPLATE_VARIABLES.OUTPUT,
   TEMPLATE_VARIABLES.EXPECTED_OUTPUT
 ]);
+var DEPRECATED_TEMPLATE_VARIABLES = /* @__PURE__ */ new Map([
+  [TEMPLATE_VARIABLES.INPUT_TEXT, TEMPLATE_VARIABLES.INPUT],
+  [TEMPLATE_VARIABLES.OUTPUT_TEXT, TEMPLATE_VARIABLES.OUTPUT],
+  [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT, TEMPLATE_VARIABLES.EXPECTED_OUTPUT]
+]);
 
 // src/evaluation/validation/prompt-validator.ts
 var ANSI_YELLOW2 = "\x1B[33m";
@@ -720,16 +732,29 @@ function validateTemplateVariables(content, source) {
     }
     match = variablePattern.exec(content);
   }
-  const hasCandidateAnswer = foundVariables.has(TEMPLATE_VARIABLES.OUTPUT_TEXT);
+  const hasCandidateAnswer = foundVariables.has(TEMPLATE_VARIABLES.OUTPUT) || foundVariables.has(TEMPLATE_VARIABLES.OUTPUT_TEXT);
   const hasExpectedOutput = foundVariables.has(TEMPLATE_VARIABLES.EXPECTED_OUTPUT);
   const hasRequiredFields = hasCandidateAnswer || hasExpectedOutput;
   if (!hasRequiredFields) {
     throw new Error(
       `Missing required fields. Must include at least one of:
-  - {{ ${TEMPLATE_VARIABLES.OUTPUT_TEXT} }}
+  - {{ ${TEMPLATE_VARIABLES.OUTPUT} }}
   - {{ ${TEMPLATE_VARIABLES.EXPECTED_OUTPUT} }}`
     );
   }
+  const deprecatedUsed = [];
+  for (const [deprecated, replacement] of DEPRECATED_TEMPLATE_VARIABLES) {
+    if (foundVariables.has(deprecated)) {
+      deprecatedUsed.push(`{{ ${deprecated} }} \u2192 {{ ${replacement} }}`);
+    }
+  }
+  if (deprecatedUsed.length > 0) {
+    console.warn(
+      `${ANSI_YELLOW2}Warning: Template at ${source} uses deprecated variable names:
+  ${deprecatedUsed.join("\n  ")}
+  These still work but will be removed in a future version.${ANSI_RESET3}`
+    );
+  }
   if (invalidVariables.length > 0) {
     const warningMessage = `${ANSI_YELLOW2}Warning: Custom evaluator template at ${source}
   Contains invalid variables: ${invalidVariables.map((v) => `{{ ${v} }}`).join(", ")}
@@ -2147,6 +2172,19 @@ function asString2(value) {
 }
 
 // src/evaluation/loaders/message-processor.ts
+var IMAGE_MEDIA_TYPES = {
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".gif": "image/gif",
+  ".webp": "image/webp",
+  ".svg": "image/svg+xml",
+  ".bmp": "image/bmp"
+};
+function detectImageMediaType(filePath) {
+  const ext = path5.extname(filePath).toLowerCase();
+  return IMAGE_MEDIA_TYPES[ext];
+}
 var ANSI_YELLOW4 = "\x1B[33m";
 var ANSI_RESET5 = "\x1B[0m";
 async function processMessages(options) {
@@ -2212,6 +2250,47 @@ async function processMessages(options) {
         }
         continue;
       }
+      if (segmentType === "image") {
+        const rawValue = asString3(rawSegment.value);
+        if (!rawValue) {
+          continue;
+        }
+        const { displayPath, resolvedPath, attempted } = await resolveFileReference2(
+          rawValue,
+          searchRoots
+        );
+        if (!resolvedPath) {
+          const attempts = attempted.length ? ["  Tried:", ...attempted.map((candidate) => `    ${candidate}`)] : void 0;
+          const context = messageType === "input" ? "" : " in expected_output";
+          logWarning3(`Image file not found${context}: ${displayPath}`, attempts);
+          continue;
+        }
+        const mediaType = detectImageMediaType(resolvedPath);
+        if (!mediaType) {
+          logWarning3(
+            `Unsupported image extension for ${displayPath}. Supported: ${Object.keys(IMAGE_MEDIA_TYPES).join(", ")}`
+          );
+          continue;
+        }
+        try {
+          const imageBuffer = await readFile4(resolvedPath);
+          const base64 = imageBuffer.toString("base64");
+          processedContent.push({
+            type: "image",
+            media_type: mediaType,
+            source: `data:${mediaType};base64,${base64}`
+          });
+          if (verbose) {
+            const label = messageType === "input" ? "[Image]" : "[Expected Output Image]";
+            console.log(`  ${label} Found: ${displayPath}`);
+            console.log(`    Resolved to: ${resolvedPath} (${mediaType})`);
+          }
+        } catch (error) {
+          const context = messageType === "input" ? "" : " expected output";
+          logWarning3(`Could not read${context} image ${resolvedPath}: ${error.message}`);
+        }
+        continue;
+      }
       const clonedSegment = cloneJsonObject(rawSegment);
       processedContent.push(clonedSegment);
       const inlineValue = clonedSegment.value;
@@ -2289,6 +2368,46 @@ async function processExpectedMessages(options) {
           }
           continue;
         }
+        if (segmentType === "image") {
+          const rawValue = asString3(rawSegment.value);
+          if (!rawValue) {
+            continue;
+          }
+          const { displayPath, resolvedPath, attempted } = await resolveFileReference2(
+            rawValue,
+            searchRoots
+          );
+          if (!resolvedPath) {
+            const attempts = attempted.length ? ["  Tried:", ...attempted.map((candidate) => `    ${candidate}`)] : void 0;
+            logWarning3(`Image file not found in expected_output: ${displayPath}`, attempts);
+            continue;
+          }
+          const mediaType = detectImageMediaType(resolvedPath);
+          if (!mediaType) {
+            logWarning3(
+              `Unsupported image extension for ${displayPath}. Supported: ${Object.keys(IMAGE_MEDIA_TYPES).join(", ")}`
+            );
+            continue;
+          }
+          try {
+            const imageBuffer = await readFile4(resolvedPath);
+            const base64 = imageBuffer.toString("base64");
+            processedContent.push({
+              type: "image",
+              media_type: mediaType,
+              source: `data:${mediaType};base64,${base64}`
+            });
+            if (verbose) {
+              console.log(`  [Expected Output Image] Found: ${displayPath}`);
+              console.log(`    Resolved to: ${resolvedPath} (${mediaType})`);
+            }
+          } catch (error) {
+            logWarning3(
+              `Could not read expected output image ${resolvedPath}: ${error.message}`
+            );
+          }
+          continue;
+        }
         processedContent.push(cloneJsonObject(rawSegment));
       }
       segment.content = processedContent;
@@ -2535,7 +2654,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
     const userFilePaths = collectResolvedInputFilePaths(inputMessages);
     const testCase = {
       id,
-      eval_set: evalSetName,
+      dataset: evalSetName,
       conversation_id: conversationId,
       question,
       input: inputMessages,
@@ -2806,7 +2925,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
   }
   const suite = interpolated;
   const evalSetNameFromSuite = asString5(suite.name)?.trim();
-  const fallbackEvalSet = path7.basename(absoluteTestPath).replace(/\.ya?ml$/i, "") || "eval";
+  const fallbackEvalSet = path7.basename(absoluteTestPath).replace(/\.eval\.ya?ml$/i, "").replace(/\.ya?ml$/i, "") || "eval";
   const evalSetName = evalSetNameFromSuite && evalSetNameFromSuite.length > 0 ? evalSetNameFromSuite : fallbackEvalSet;
   const rawTestcases = resolveTests(suite);
   const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm-grader";
@@ -2927,7 +3046,8 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
     const caseTargets = extractTargetsFromTestCase(evalcase);
     const testCase = {
       id,
-      eval_set: evalSetName,
+      dataset: evalSetName,
+      category: options?.category,
       conversation_id: conversationId,
       question,
       input: inputMessages,
@@ -3851,6 +3971,48 @@ import { createWriteStream } from "node:fs";
 import { mkdir } from "node:fs/promises";
 import path10 from "node:path";
 
+// src/evaluation/providers/claude-content.ts
+function toContentArray(content) {
+  if (!Array.isArray(content)) return void 0;
+  let hasNonText = false;
+  const blocks = [];
+  for (const part of content) {
+    if (!part || typeof part !== "object") continue;
+    const p = part;
+    if (p.type === "text" && typeof p.text === "string") {
+      blocks.push({ type: "text", text: p.text });
+    } else if (p.type === "image" && typeof p.source === "object" && p.source !== null) {
+      const src = p.source;
+      const mediaType = typeof p.media_type === "string" ? p.media_type : typeof src.media_type === "string" ? src.media_type : "application/octet-stream";
+      const data = typeof src.data === "string" ? `data:${mediaType};base64,${src.data}` : typeof p.url === "string" ? p.url : "";
+      blocks.push({ type: "image", media_type: mediaType, source: data });
+      hasNonText = true;
+    } else if (p.type === "tool_use") {
+    } else if (p.type === "tool_result") {
+    }
+  }
+  return hasNonText && blocks.length > 0 ? blocks : void 0;
+}
+function extractTextContent(content) {
+  if (typeof content === "string") {
+    return content;
+  }
+  if (!Array.isArray(content)) {
+    return void 0;
+  }
+  const textParts = [];
+  for (const part of content) {
+    if (!part || typeof part !== "object") {
+      continue;
+    }
+    const p = part;
+    if (p.type === "text" && typeof p.text === "string") {
+      textParts.push(p.text);
+    }
+  }
+  return textParts.length > 0 ? textParts.join("\n") : void 0;
+}
+
 // src/evaluation/providers/claude-log-tracker.ts
 var GLOBAL_LOGS_KEY = Symbol.for("agentv.claudeLogs");
 var GLOBAL_SUBSCRIBERS_KEY = Symbol.for("agentv.claudeLogSubscribers");
@@ -4016,11 +4178,12 @@ var ClaudeCliProvider = class {
             if (betaMessage && typeof betaMessage === "object") {
               const msg = betaMessage;
               const content = msg.content;
+              const structuredContent = toContentArray(content);
               const textContent = extractTextContent(content);
               const toolCalls = extractToolCalls(content);
               const outputMsg = {
                 role: "assistant",
-                content: textContent,
+                content: structuredContent ?? textContent,
                 toolCalls: toolCalls.length > 0 ? toolCalls : void 0
               };
               output.push(outputMsg);
@@ -4359,25 +4522,6 @@ function summarizeEvent(event) {
       return void 0;
   }
 }
-function extractTextContent(content) {
-  if (typeof content === "string") {
-    return content;
-  }
-  if (!Array.isArray(content)) {
-    return void 0;
-  }
-  const textParts = [];
-  for (const part of content) {
-    if (!part || typeof part !== "object") {
-      continue;
-    }
-    const p = part;
-    if (p.type === "text" && typeof p.text === "string") {
-      textParts.push(p.text);
-    }
-  }
-  return textParts.length > 0 ? textParts.join("\n") : void 0;
-}
 function extractToolCalls(content) {
   if (!Array.isArray(content)) {
     return [];
@@ -4550,11 +4694,12 @@ var ClaudeSdkProvider = class {
             if (betaMessage && typeof betaMessage === "object") {
               const msg = betaMessage;
               const content = msg.content;
-              const textContent = extractTextContent2(content);
+              const structuredContent = toContentArray(content);
+              const textContent = extractTextContent(content);
               const toolCalls = extractToolCalls2(content);
               const outputMsg = {
                 role: "assistant",
-                content: textContent,
+                content: structuredContent ?? textContent,
                 toolCalls: toolCalls.length > 0 ? toolCalls : void 0
               };
               output.push(outputMsg);
@@ -4672,25 +4817,6 @@ var ClaudeSdkProvider = class {
     }
   }
 };
-function extractTextContent2(content) {
-  if (typeof content === "string") {
-    return content;
-  }
-  if (!Array.isArray(content)) {
-    return void 0;
-  }
-  const textParts = [];
-  for (const part of content) {
-    if (!part || typeof part !== "object") {
-      continue;
-    }
-    const p = part;
-    if (p.type === "text" && typeof p.text === "string") {
-      textParts.push(p.text);
-    }
-  }
-  return textParts.length > 0 ? textParts.join("\n") : void 0;
-}
 function extractToolCalls2(content) {
   if (!Array.isArray(content)) {
     return [];
@@ -4914,7 +5040,7 @@ function convertMessages(messages) {
   return messages.map((msg) => ({
     role: msg.role,
     name: msg.name,
-    content: msg.content,
+    content: isContentArray(msg.content) ? msg.content : typeof msg.content === "string" ? msg.content : void 0,
     toolCalls: msg.tool_calls?.map((tc) => ({
       tool: tc.tool,
       input: tc.input,
@@ -7167,6 +7293,35 @@ function extractPiTextContent(content) {
   }
   return textParts.length > 0 ? textParts.join("\n") : void 0;
 }
+function toPiContentArray(content) {
+  if (!Array.isArray(content)) return void 0;
+  let hasNonText = false;
+  const blocks = [];
+  for (const part of content) {
+    if (!part || typeof part !== "object") continue;
+    const p = part;
+    if (p.type === "text" && typeof p.text === "string") {
+      blocks.push({ type: "text", text: p.text });
+    } else if (p.type === "image") {
+      const mediaType = typeof p.media_type === "string" ? p.media_type : "application/octet-stream";
+      let source = "";
+      if (typeof p.source === "object" && p.source !== null) {
+        const src = p.source;
+        const srcMediaType = typeof src.media_type === "string" ? src.media_type : mediaType;
+        source = typeof src.data === "string" ? `data:${srcMediaType};base64,${src.data}` : "";
+      }
+      if (!source && typeof p.url === "string") {
+        source = p.url;
+      }
+      if (source) {
+        blocks.push({ type: "image", media_type: mediaType, source });
+        hasNonText = true;
+      }
+    } else if (p.type === "tool_use" || p.type === "tool_result") {
+    }
+  }
+  return hasNonText && blocks.length > 0 ? blocks : void 0;
+}
 function toFiniteNumber(value) {
   if (typeof value === "number" && Number.isFinite(value)) return value;
   return void 0;
@@ -8337,7 +8492,8 @@ function convertAgentMessage(message, toolTrackers, completedToolResults) {
   }
   const msg = message;
   const role = typeof msg.role === "string" ? msg.role : "unknown";
-  const content = extractPiTextContent(msg.content);
+  const structuredContent = toPiContentArray(msg.content);
+  const content = structuredContent ?? extractPiTextContent(msg.content);
   const toolCalls = extractToolCalls4(msg.content, toolTrackers, completedToolResults);
   const startTimeVal = typeof msg.timestamp === "number" ? new Date(msg.timestamp).toISOString() : typeof msg.timestamp === "string" ? msg.timestamp : void 0;
   let msgTokenUsage;
@@ -10187,13 +10343,13 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
 async function execShellWithStdin(command, stdinPayload, options = {}) {
   const { mkdir: mkdir16, readFile: readFile14, rm: rm6, writeFile: writeFile9 } = await import("node:fs/promises");
   const { tmpdir: tmpdir3 } = await import("node:os");
-  const path47 = await import("node:path");
+  const path48 = await import("node:path");
   const { randomUUID: randomUUID10 } = await import("node:crypto");
-  const dir = path47.join(tmpdir3(), `agentv-exec-${randomUUID10()}`);
+  const dir = path48.join(tmpdir3(), `agentv-exec-${randomUUID10()}`);
   await mkdir16(dir, { recursive: true });
-  const stdinPath = path47.join(dir, "stdin.txt");
-  const stdoutPath = path47.join(dir, "stdout.txt");
-  const stderrPath = path47.join(dir, "stderr.txt");
+  const stdinPath = path48.join(dir, "stdin.txt");
+  const stdoutPath = path48.join(dir, "stdout.txt");
+  const stderrPath = path48.join(dir, "stderr.txt");
   await writeFile9(stdinPath, stdinPayload, "utf8");
   const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
   const { spawn: spawn5 } = await import("node:child_process");
@@ -10509,6 +10665,56 @@ function toCamelCaseDeep(obj) {
 
 // src/evaluation/evaluators/code-evaluator.ts
 var FILE_BACKED_OUTPUT_THRESHOLD = 5e4;
+var DATA_URI_RE = /^data:([^;]+);base64,(.+)$/s;
+async function materializeContentForGrader(messages, getWorkDir) {
+  if (!messages || messages.length === 0) return messages ?? null;
+  let hasAnyImage = false;
+  for (const msg of messages) {
+    if (isContentArray(msg.content)) {
+      for (const block of msg.content) {
+        if (block.type === "image") {
+          hasAnyImage = true;
+          break;
+        }
+      }
+    }
+    if (hasAnyImage) break;
+  }
+  if (!hasAnyImage) return messages;
+  let counter = 0;
+  const result = [];
+  for (const msg of messages) {
+    if (!isContentArray(msg.content)) {
+      result.push(msg);
+      continue;
+    }
+    if (!msg.content.some((b) => b.type === "image")) {
+      result.push(msg);
+      continue;
+    }
+    const blocks = [];
+    for (const block of msg.content) {
+      if (block.type !== "image") {
+        blocks.push({ ...block });
+        continue;
+      }
+      const img = block;
+      const match = DATA_URI_RE.exec(img.source);
+      if (match) {
+        const [, mediaType, base64Data] = match;
+        const ext = mediaType.split("/")[1] === "jpeg" ? "jpg" : mediaType.split("/")[1] ?? "bin";
+        const dir = await getWorkDir();
+        const filePath = join(dir, `img-${counter++}.${ext}`);
+        await writeFile6(filePath, Buffer.from(base64Data, "base64"));
+        blocks.push({ type: "image", media_type: img.media_type, path: filePath });
+      } else {
+        blocks.push({ type: "image", media_type: img.media_type, path: img.source });
+      }
+    }
+    result.push({ ...msg, content: blocks });
+  }
+  return result;
+}
 var CodeEvaluator = class {
   kind = "code-grader";
   command;
@@ -10524,7 +10730,18 @@ var CodeEvaluator = class {
     this.target = options.target;
   }
   async evaluate(context) {
-    let outputForPayload = context.output ?? null;
+    let imageTmpDir;
+    const getImageDir = async () => {
+      if (!imageTmpDir) {
+        imageTmpDir = await mkdtemp2(join(tmpdir2(), "agentv-img-"));
+      }
+      return imageTmpDir;
+    };
+    const materializedOutput = await materializeContentForGrader(
+      context.output,
+      getImageDir
+    );
+    let outputForPayload = materializedOutput;
     let outputPath;
     if (outputForPayload) {
       const serialized = JSON.stringify(outputForPayload);
@@ -10537,12 +10754,17 @@ var CodeEvaluator = class {
     }
     const payload = {
       criteria: context.evalCase.criteria,
-      expectedOutput: context.evalCase.expected_output,
-      outputText: context.candidate,
+      expectedOutput: await materializeContentForGrader(
+        context.evalCase.expected_output,
+        getImageDir
+      ),
       output: outputForPayload,
       outputPath,
       inputFiles: context.evalCase.file_paths,
-      input: context.evalCase.input,
+      input: await materializeContentForGrader(
+        context.evalCase.input,
+        getImageDir
+      ),
       trace: context.trace ?? null,
       tokenUsage: context.tokenUsage ?? null,
       costUsd: context.costUsd ?? null,
@@ -10551,9 +10773,7 @@ var CodeEvaluator = class {
       endTime: context.endTime ?? null,
       fileChanges: context.fileChanges ?? null,
       workspacePath: context.workspacePath ?? null,
-      config: this.config ?? null,
-      inputText: context.evalCase.question,
-      expectedOutputText: context.evalCase.reference_answer ?? ""
+      config: this.config ?? null
     };
     const inputPayload = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
     let proxyEnv;
@@ -10643,6 +10863,10 @@ var CodeEvaluator = class {
         await rm3(dirname(outputPath), { recursive: true, force: true }).catch(() => {
         });
       }
+      if (imageTmpDir) {
+        await rm3(imageTmpDir, { recursive: true, force: true }).catch(() => {
+        });
+      }
     }
   }
 };
@@ -10720,13 +10944,13 @@ Be concise and focused in your evaluation. Provide succinct, specific feedback r
 {{${TEMPLATE_VARIABLES.CRITERIA}}}
 
 [[ ## question ## ]]
-{{${TEMPLATE_VARIABLES.INPUT_TEXT}}}
+{{${TEMPLATE_VARIABLES.INPUT}}}
 
 [[ ## reference_answer ## ]]
-{{${TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT}}}
+{{${TEMPLATE_VARIABLES.EXPECTED_OUTPUT}}}
 
 [[ ## answer ## ]]
-{{${TEMPLATE_VARIABLES.OUTPUT_TEXT}}}`;
+{{${TEMPLATE_VARIABLES.OUTPUT}}}`;
 var freeformEvaluationSchema = z3.object({
   score: z3.number().min(0).max(1).describe("Score between 0.0 and 1.0"),
   assertions: z3.array(
@@ -10798,21 +11022,19 @@ var LlmGraderEvaluator = class {
   async evaluateFreeform(context, graderProvider) {
     const formattedQuestion = context.promptInputs.question && context.promptInputs.question.trim().length > 0 ? context.promptInputs.question : context.evalCase.question;
     const variables = {
-      [TEMPLATE_VARIABLES.INPUT]: JSON.stringify(context.evalCase.input, null, 2),
-      [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: JSON.stringify(
-        context.evalCase.expected_output,
-        null,
-        2
-      ),
-      [TEMPLATE_VARIABLES.OUTPUT]: JSON.stringify(context.output ?? [], null, 2),
+      [TEMPLATE_VARIABLES.INPUT]: formattedQuestion.trim(),
+      [TEMPLATE_VARIABLES.OUTPUT]: context.candidate.trim(),
+      [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.CRITERIA]: context.evalCase.criteria.trim(),
       [TEMPLATE_VARIABLES.FILE_CHANGES]: context.fileChanges ?? "",
+      // Deprecated aliases — same values as the primary variables above
       [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
       [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context.candidate.trim(),
       [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context.evalCase.reference_answer ?? "").trim()
     };
     const systemPrompt = buildOutputSchema();
     const evaluatorTemplate = context.evaluatorTemplateOverride ?? this.evaluatorTemplate ?? DEFAULT_EVALUATOR_TEMPLATE;
+    warnDeprecatedTemplateVars(evaluatorTemplate);
     let userPrompt = substituteVariables(evaluatorTemplate, variables);
     if (context.fileChanges && !context.evaluatorTemplateOverride && !this.evaluatorTemplate) {
       userPrompt += `
@@ -10824,13 +11046,15 @@ ${context.fileChanges}`;
       userPrompt,
       systemPrompt
     };
+    const images = context.output ? extractImageBlocks(context.output) : [];
     try {
       const { data, tokenUsage } = await this.runWithRetry({
         context,
         graderProvider,
         systemPrompt,
         userPrompt,
-        schema: freeformEvaluationSchema
+        schema: freeformEvaluationSchema,
+        images
       });
       const score = clampScore(data.score);
       const assertions = Array.isArray(data.assertions) ? data.assertions.slice(0, 8) : [];
@@ -10874,13 +11098,15 @@ ${context.fileChanges}`;
       userPrompt: prompt,
       systemPrompt
     };
+    const images = context.output ? extractImageBlocks(context.output) : [];
     try {
       const { data, tokenUsage } = await this.runWithRetry({
         context,
         graderProvider,
         systemPrompt,
         userPrompt: prompt,
-        schema: rubricEvaluationSchema
+        schema: rubricEvaluationSchema,
+        images
       });
       const { score, verdict, assertions } = calculateRubricScore(data, rubrics);
       return {
@@ -10917,13 +11143,15 @@ ${context.fileChanges}`;
       userPrompt: prompt,
       systemPrompt
     };
+    const images = context.output ? extractImageBlocks(context.output) : [];
     try {
       const { data, tokenUsage } = await this.runWithRetry({
         context,
         graderProvider,
         systemPrompt,
         userPrompt: prompt,
-        schema: scoreRangeEvaluationSchema
+        schema: scoreRangeEvaluationSchema,
+        images
       });
       const { score, verdict, assertions, details } = calculateScoreRangeResult(data, rubrics);
       return {
@@ -11130,12 +11358,17 @@ ${context.fileChanges}`;
     const formattedQuestion = context.promptInputs.question && context.promptInputs.question.trim().length > 0 ? context.promptInputs.question : context.evalCase.question;
     const variables = {
       [TEMPLATE_VARIABLES.CRITERIA]: context.evalCase.criteria.trim(),
+      [TEMPLATE_VARIABLES.INPUT]: formattedQuestion.trim(),
+      [TEMPLATE_VARIABLES.OUTPUT]: context.candidate.trim(),
+      [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context.evalCase.reference_answer ?? "").trim(),
+      [TEMPLATE_VARIABLES.FILE_CHANGES]: context.fileChanges ?? "",
+      // Deprecated aliases
       [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
       [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context.candidate.trim(),
-      [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context.evalCase.reference_answer ?? "").trim(),
-      [TEMPLATE_VARIABLES.FILE_CHANGES]: context.fileChanges ?? ""
+      [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context.evalCase.reference_answer ?? "").trim()
     };
     if (this.evaluatorTemplate) {
+      warnDeprecatedTemplateVars(this.evaluatorTemplate);
       return substituteVariables(this.evaluatorTemplate, variables);
     }
     const config = context.evaluator;
@@ -11186,11 +11419,16 @@ ${context.fileChanges}`;
     if (this.evaluatorTemplate) {
       const variables = {
         [TEMPLATE_VARIABLES.CRITERIA]: context.evalCase.criteria.trim(),
+        [TEMPLATE_VARIABLES.INPUT]: formattedQuestion.trim(),
+        [TEMPLATE_VARIABLES.OUTPUT]: context.candidate.trim(),
+        [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context.evalCase.reference_answer ?? "").trim(),
+        [TEMPLATE_VARIABLES.FILE_CHANGES]: context.fileChanges ?? "",
+        // Deprecated aliases
         [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
         [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context.candidate.trim(),
-        [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context.evalCase.reference_answer ?? "").trim(),
-        [TEMPLATE_VARIABLES.FILE_CHANGES]: context.fileChanges ?? ""
+        [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context.evalCase.reference_answer ?? "").trim()
       };
+      warnDeprecatedTemplateVars(this.evaluatorTemplate);
       const customPrompt = substituteVariables(this.evaluatorTemplate, variables);
       const outputSchema = rubrics && rubrics.length > 0 ? buildRubricOutputSchema() : buildOutputSchema();
       return `${customPrompt}
@@ -11361,18 +11599,35 @@ ${outputSchema}`;
   // LLM mode retry logic
   // ---------------------------------------------------------------------------
   async runWithRetry(options) {
-    const { context, graderProvider, systemPrompt, userPrompt, schema } = options;
+    const { context, graderProvider, systemPrompt, userPrompt, schema, images } = options;
     let lastError;
     for (let attempt = 1; attempt <= 3; attempt++) {
       try {
         const model = graderProvider.asLanguageModel?.();
         if (model) {
-          const result = await generateText2({
+          const modelOptions = {
+            ...this.maxOutputTokens ? { maxTokens: this.maxOutputTokens } : {},
+            ...typeof this.temperature === "number" ? { temperature: this.temperature } : {}
+          };
+          const hasImages = images && images.length > 0;
+          const result = hasImages ? await generateText2({
+            model,
+            system: systemPrompt,
+            messages: [
+              {
+                role: "user",
+                content: [
+                  { type: "text", text: userPrompt },
+                  ...toAiSdkImageParts(images)
+                ]
+              }
+            ],
+            ...modelOptions
+          }) : await generateText2({
             model,
             system: systemPrompt,
             prompt: userPrompt,
-            ...this.maxOutputTokens ? { maxTokens: this.maxOutputTokens } : {},
-            ...typeof this.temperature === "number" ? { temperature: this.temperature } : {}
+            ...modelOptions
           });
           const data2 = schema.parse(parseJsonFromText(result.text));
           const rawUsage = result.usage;
@@ -11432,6 +11687,26 @@ function substituteVariables(template, variables) {
     return variables[varName] ?? match;
   });
 }
+var ANSI_YELLOW7 = "\x1B[33m";
+var ANSI_RESET8 = "\x1B[0m";
+var warnedTemplateStrings = /* @__PURE__ */ new Set();
+function warnDeprecatedTemplateVars(template) {
+  if (warnedTemplateStrings.has(template)) return;
+  const used = [];
+  for (const [deprecated, replacement] of DEPRECATED_TEMPLATE_VARIABLES) {
+    if (new RegExp(`\\{\\{\\s*${deprecated}\\s*\\}\\}`).test(template)) {
+      used.push(`{{ ${deprecated} }} \u2192 {{ ${replacement} }}`);
+    }
+  }
+  if (used.length > 0) {
+    warnedTemplateStrings.add(template);
+    console.warn(
+      `${ANSI_YELLOW7}\u26A0 Deprecated template variables detected (they still work but will be removed in a future version):
+  ${used.join("\n  ")}
+  Update your custom evaluator template to use the new names.${ANSI_RESET8}`
+    );
+  }
+}
 function calculateRubricScore(result, rubrics) {
   const rubricMap = new Map(rubrics.map((rubric) => [rubric.id, rubric]));
   const assertions = [];
@@ -11526,6 +11801,26 @@ function calculateScoreRangeResult(result, rubrics) {
     }
   };
 }
+function extractImageBlocks(messages) {
+  const images = [];
+  for (const msg of messages) {
+    if (msg.role !== "assistant") continue;
+    if (!isContentArray(msg.content)) continue;
+    for (const block of msg.content) {
+      if (block.type === "image") {
+        images.push(block);
+      }
+    }
+  }
+  return images;
+}
+function toAiSdkImageParts(images) {
+  return images.map((img) => ({
+    type: "image",
+    image: img.source,
+    mediaType: img.media_type || void 0
+  }));
+}
 function resolveSandboxed(basePath, relativePath) {
   const resolved = path35.resolve(basePath, relativePath);
   if (!resolved.startsWith(basePath + path35.sep) && resolved !== basePath) {
@@ -12267,115 +12562,115 @@ var FieldAccuracyEvaluator = class {
    * Evaluate a single field against the expected value.
    */
   evaluateField(fieldConfig, candidateData, expectedData) {
-    const { path: path47, match, required = true, weight = 1 } = fieldConfig;
-    const candidateValue = resolvePath(candidateData, path47);
-    const expectedValue = resolvePath(expectedData, path47);
+    const { path: path48, match, required = true, weight = 1 } = fieldConfig;
+    const candidateValue = resolvePath(candidateData, path48);
+    const expectedValue = resolvePath(expectedData, path48);
     if (expectedValue === void 0) {
       return {
-        path: path47,
+        path: path48,
         score: 1,
         // No expected value means no comparison needed
         weight,
         hit: true,
-        message: `${path47}: no expected value`
+        message: `${path48}: no expected value`
       };
     }
     if (candidateValue === void 0) {
       if (required) {
         return {
-          path: path47,
+          path: path48,
           score: 0,
           weight,
           hit: false,
-          message: `${path47} (required, missing)`
+          message: `${path48} (required, missing)`
         };
       }
       return {
-        path: path47,
+        path: path48,
         score: 1,
         // Don't penalize missing optional fields
         weight: 0,
         // Zero weight means it won't affect the score
         hit: true,
-        message: `${path47}: optional field missing`
+        message: `${path48}: optional field missing`
       };
     }
     switch (match) {
       case "exact":
-        return this.compareExact(path47, candidateValue, expectedValue, weight);
+        return this.compareExact(path48, candidateValue, expectedValue, weight);
       case "numeric_tolerance":
         return this.compareNumericTolerance(
-          path47,
+          path48,
           candidateValue,
           expectedValue,
           fieldConfig,
           weight
         );
       case "date":
-        return this.compareDate(path47, candidateValue, expectedValue, fieldConfig, weight);
+        return this.compareDate(path48, candidateValue, expectedValue, fieldConfig, weight);
       default:
         return {
-          path: path47,
+          path: path48,
           score: 0,
           weight,
           hit: false,
-          message: `${path47}: unknown match type "${match}"`
+          message: `${path48}: unknown match type "${match}"`
         };
     }
   }
   /**
    * Exact equality comparison.
    */
-  compareExact(path47, candidateValue, expectedValue, weight) {
+  compareExact(path48, candidateValue, expectedValue, weight) {
     if (deepEqual(candidateValue, expectedValue)) {
       return {
-        path: path47,
+        path: path48,
         score: 1,
         weight,
         hit: true,
-        message: path47
+        message: path48
       };
     }
     if (typeof candidateValue !== typeof expectedValue) {
       return {
-        path: path47,
+        path: path48,
         score: 0,
         weight,
         hit: false,
-        message: `${path47} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
+        message: `${path48} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
       };
     }
     return {
-      path: path47,
+      path: path48,
       score: 0,
       weight,
       hit: false,
-      message: `${path47} (value mismatch)`
+      message: `${path48} (value mismatch)`
     };
   }
   /**
    * Numeric comparison with absolute or relative tolerance.
    */
-  compareNumericTolerance(path47, candidateValue, expectedValue, fieldConfig, weight) {
+  compareNumericTolerance(path48, candidateValue, expectedValue, fieldConfig, weight) {
     const { tolerance = 0, relative = false } = fieldConfig;
     const candidateNum = toNumber(candidateValue);
     const expectedNum = toNumber(expectedValue);
     if (candidateNum === null || expectedNum === null) {
       return {
-        path: path47,
+        path: path48,
         score: 0,
         weight,
         hit: false,
-        message: `${path47} (non-numeric value)`
+        message: `${path48} (non-numeric value)`
       };
     }
     if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
       return {
-        path: path47,
+        path: path48,
         score: 0,
         weight,
         hit: false,
-        message: `${path47} (invalid numeric value)`
+        message: `${path48} (invalid numeric value)`
       };
     }
     const diff = Math.abs(candidateNum - expectedNum);
@@ -12388,61 +12683,61 @@ var FieldAccuracyEvaluator = class {
     }
     if (withinTolerance) {
       return {
-        path: path47,
+        path: path48,
         score: 1,
         weight,
         hit: true,
-        message: `${path47} (within tolerance: diff=${diff.toFixed(2)})`
+        message: `${path48} (within tolerance: diff=${diff.toFixed(2)})`
       };
     }
     return {
-      path: path47,
+      path: path48,
       score: 0,
       weight,
       hit: false,
-      message: `${path47} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
+      message: `${path48} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
     };
   }
   /**
    * Date comparison with format normalization.
    */
-  compareDate(path47, candidateValue, expectedValue, fieldConfig, weight) {
+  compareDate(path48, candidateValue, expectedValue, fieldConfig, weight) {
     const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
     const candidateDate = parseDate(String(candidateValue), formats);
     const expectedDate = parseDate(String(expectedValue), formats);
     if (candidateDate === null) {
       return {
-        path: path47,
+        path: path48,
         score: 0,
         weight,
         hit: false,
-        message: `${path47} (unparseable candidate date)`
+        message: `${path48} (unparseable candidate date)`
       };
     }
     if (expectedDate === null) {
       return {
-        path: path47,
+        path: path48,
         score: 0,
         weight,
         hit: false,
-        message: `${path47} (unparseable expected date)`
+        message: `${path48} (unparseable expected date)`
       };
     }
     if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
       return {
-        path: path47,
+        path: path48,
         score: 1,
         weight,
         hit: true,
-        message: path47
+        message: path48
       };
     }
     return {
-      path: path47,
+      path: path48,
       score: 0,
       weight,
       hit: false,
-      message: `${path47} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
+      message: `${path48} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
     };
   }
   /**
@@ -12475,11 +12770,11 @@ var FieldAccuracyEvaluator = class {
     };
   }
 };
-function resolvePath(obj, path47) {
-  if (!path47 || !obj) {
+function resolvePath(obj, path48) {
+  if (!path48 || !obj) {
     return void 0;
   }
-  const parts = path47.split(/\.|\[|\]/).filter((p) => p.length > 0);
+  const parts = path48.split(/\.|\[|\]/).filter((p) => p.length > 0);
   let current = obj;
   for (const part of parts) {
     if (current === null || current === void 0) {
@@ -12771,11 +13066,12 @@ function assembleLlmGraderPrompt(input) {
 function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, evaluatorTemplateOverride) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const variables = {
-    [TEMPLATE_VARIABLES.INPUT]: JSON.stringify(evalCase.input, null, 2),
-    [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: JSON.stringify(evalCase.expected_output, null, 2),
-    [TEMPLATE_VARIABLES.OUTPUT]: JSON.stringify([], null, 2),
+    [TEMPLATE_VARIABLES.INPUT]: formattedQuestion.trim(),
+    [TEMPLATE_VARIABLES.OUTPUT]: candidate.trim(),
+    [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (evalCase.reference_answer ?? "").trim(),
     [TEMPLATE_VARIABLES.CRITERIA]: evalCase.criteria.trim(),
     [TEMPLATE_VARIABLES.FILE_CHANGES]: fileChanges ?? "",
+    // Deprecated aliases
     [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
     [TEMPLATE_VARIABLES.OUTPUT_TEXT]: candidate.trim(),
     [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (evalCase.reference_answer ?? "").trim()
@@ -12962,8 +13258,8 @@ var TokenUsageEvaluator = class {
 };
 
 // src/evaluation/evaluators/tool-trajectory.ts
-function getNestedValue(obj, path47) {
-  const parts = path47.split(".");
+function getNestedValue(obj, path48) {
+  const parts = path48.split(".");
   let current = obj;
   for (const part of parts) {
     if (current === null || current === void 0 || typeof current !== "object") {
@@ -13832,16 +14128,13 @@ async function executePromptTemplate(script, context, config, timeoutMs) {
   const payload = {
     criteria: context.evalCase.criteria,
     expectedOutput: context.evalCase.expected_output,
-    outputText: context.candidate,
     output: context.output ?? null,
     inputFiles: context.evalCase.file_paths,
     input: context.evalCase.input,
     trace: context.trace ?? null,
     fileChanges: context.fileChanges ?? null,
     workspacePath: context.workspacePath ?? null,
-    config: config ?? context.config ?? null,
-    inputText: context.evalCase.question,
-    expectedOutputText: context.evalCase.reference_answer ?? ""
+    config: config ?? context.config ?? null
   };
   const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
   const scriptPath = script[script.length - 1];
@@ -15521,7 +15814,8 @@ async function runEvaluation(options) {
           const budgetResult = {
             timestamp: (now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
             testId: evalCase.id,
-            eval_set: evalCase.eval_set,
+            dataset: evalCase.dataset,
+            category: evalCase.category,
             score: 0,
             assertions: [],
             output: [],
@@ -15557,7 +15851,8 @@ async function runEvaluation(options) {
           const haltResult = {
             timestamp: (now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
             testId: evalCase.id,
-            eval_set: evalCase.eval_set,
+            dataset: evalCase.dataset,
+            category: evalCase.category,
             score: 0,
             assertions: [],
             output: [],
@@ -16556,7 +16851,8 @@ async function evaluateCandidate(options) {
   return {
     timestamp: completedAt.toISOString(),
     testId: evalCase.id,
-    eval_set: evalCase.eval_set,
+    dataset: evalCase.dataset,
+    category: evalCase.category,
     conversationId: evalCase.conversation_id,
     score: score.score,
     assertions: score.assertions,
@@ -16906,7 +17202,8 @@ function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs,
   return {
     timestamp: timestamp.toISOString(),
     testId: evalCase.id,
-    eval_set: evalCase.eval_set,
+    dataset: evalCase.dataset,
+    category: evalCase.category,
     conversationId: evalCase.conversation_id,
     score: 0,
     assertions: [{ text: `Error: ${message}`, passed: false }],
@@ -17479,6 +17776,18 @@ function trimBaselineResult(result) {
   return trimmed;
 }
 
+// src/evaluation/category.ts
+import path47 from "node:path";
+var DEFAULT_CATEGORY = "Uncategorized";
+function deriveCategory(relativePath) {
+  const parts = relativePath.split(path47.sep);
+  if (parts.length <= 1) {
+    return DEFAULT_CATEGORY;
+  }
+  const dirs = parts.slice(0, -1).filter((d) => d !== "evals");
+  return dirs.length > 0 ? dirs.join("/") : DEFAULT_CATEGORY;
+}
+
 // src/observability/otel-exporter.ts
 var OTEL_BACKEND_PRESETS = {
   langfuse: {
@@ -17602,7 +17911,7 @@ var OtelTraceExporter = class {
         rootSpan.setAttribute("gen_ai.system", "agentv");
         rootSpan.setAttribute("agentv.test_id", result.testId);
         rootSpan.setAttribute("agentv.target", result.target);
-        if (result.eval_set) rootSpan.setAttribute("agentv.eval_set", result.eval_set);
+        if (result.dataset) rootSpan.setAttribute("agentv.dataset", result.dataset);
         rootSpan.setAttribute("agentv.score", result.score);
         if (captureContent && result.output.length > 0) {
           const lastMsg = result.output[result.output.length - 1];
@@ -17811,7 +18120,7 @@ var OtelStreamingObserver = class {
     this.rootSpan.setAttribute("gen_ai.system", "agentv");
     this.rootSpan.setAttribute("agentv.test_id", testId);
     this.rootSpan.setAttribute("agentv.target", target);
-    if (evalSet) this.rootSpan.setAttribute("agentv.eval_set", evalSet);
+    if (evalSet) this.rootSpan.setAttribute("agentv.dataset", evalSet);
     this.rootCtx = this.api.trace.setSpan(this.api.context.active(), this.rootSpan);
   }
   /** Create and immediately export a tool span */
@@ -17983,9 +18292,11 @@ function createAgentKernel() {
   return { status: "stub" };
 }
 export {
+  COMMON_TARGET_SETTINGS,
   CodeEvaluator,
   CompositeEvaluator,
   CostEvaluator,
+  DEFAULT_CATEGORY,
   DEFAULT_EVALUATOR_TEMPLATE,
   DEFAULT_EVAL_PATTERNS,
   DEFAULT_EXPLORATION_TOOLS,
@@ -18039,6 +18350,7 @@ export {
   createTempWorkspace,
   deepEqual,
   defineConfig,
+  deriveCategory,
   detectFormat,
   discoverAssertions,
   discoverCopilotSessions,
@@ -18052,7 +18364,9 @@ export {
   explorationRatio,
   extractCacheConfig,
   extractFailOnError,
+  extractImageBlocks,
   extractJsonBlob,
+  extractLastAssistantContent,
   extractTargetFromSuite,
   extractTargetsFromSuite,
   extractTargetsFromTestCase,
@@ -18066,12 +18380,15 @@ export {
   getAgentvHome,
   getOutputFilenames,
   getSubagentsRoot,
+  getTextContent,
   getTraceStateRoot,
   getWorkspacePath,
   getWorkspacePoolRoot,
   getWorkspacesRoot,
   initializeBaseline,
   isAgentSkillsFormat,
+  isContent,
+  isContentArray,
   isEvaluatorKind,
   isJsonObject,
   isJsonValue,