agentv 0.10.0 → 0.11.0

package/dist/{chunk-J5HK75TC.js → chunk-7CJK3EYC.js}

@@ -11752,6 +11752,33 @@ var ANSI_YELLOW = "\x1B[33m";
 var ANSI_RESET = "\x1B[0m";
 var SCHEMA_EVAL_V2 = "agentv-eval-v2";
 var SCHEMA_CONFIG_V2 = "agentv-config-v2";
+async function readTestSuiteMetadata(testFilePath) {
+  try {
+    const absolutePath = path8.resolve(testFilePath);
+    const content = await readFile3(absolutePath, "utf8");
+    const parsed = parse3(content);
+    if (!isJsonObject(parsed)) {
+      return {};
+    }
+    return { target: extractTargetFromSuite(parsed) };
+  } catch {
+    return {};
+  }
+}
+function extractTargetFromSuite(suite) {
+  const execution = suite.execution;
+  if (execution && typeof execution === "object" && !Array.isArray(execution)) {
+    const executionTarget = execution.target;
+    if (typeof executionTarget === "string" && executionTarget.trim().length > 0) {
+      return executionTarget.trim();
+    }
+  }
+  const targetValue = suite.target;
+  if (typeof targetValue === "string" && targetValue.trim().length > 0) {
+    return targetValue.trim();
+  }
+  return void 0;
+}
 async function loadConfig(evalFilePath, repoRoot) {
   const directories = buildDirectoryChain(evalFilePath, repoRoot);
   for (const directory of directories) {
@@ -11928,6 +11955,8 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
     throw new Error(`Invalid test file format: ${evalFilePath} - missing 'evalcases' field`);
   }
   const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm_judge";
+  const globalExecution = isJsonObject(suite.execution) ? suite.execution : void 0;
+  const globalTarget = asString(globalExecution?.target) ?? asString(suite.target);
   const results = [];
   for (const rawEvalcase of rawTestcases) {
     if (!isJsonObject(rawEvalcase)) {
@@ -11982,7 +12011,7 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
     const referenceAnswer = expectedContent ? await resolveAssistantContent(expectedContent, searchRoots, verbose) : "";
     const question = inputTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
     const evalCaseEvaluatorKind = coerceEvaluator(evalcase.evaluator, id) ?? globalEvaluator;
-    const evaluators = await parseEvaluators(evalcase, searchRoots, id ?? "unknown");
+    const evaluators = await parseEvaluators(evalcase, globalExecution, searchRoots, id ?? "unknown");
     const userFilePaths = [];
     for (const segment of inputSegments) {
       if (segment.type === "file" && typeof segment.resolvedPath === "string") {
@@ -12068,14 +12097,13 @@ function formatSegment(segment) {
     const text = asString(segment.text);
     const filePath = asString(segment.path);
     if (text && filePath) {
-      return `=== ${filePath} ===
-${text}`;
+      return formatFileContents([{ content: text.trim(), isFile: true, displayPath: filePath }]);
     }
   }
   return void 0;
 }
 async function buildPromptInputs(testCase) {
-  const guidelineContents = [];
+  const guidelineParts = [];
   for (const rawPath of testCase.guideline_paths) {
     const absolutePath = path8.resolve(rawPath);
     if (!await fileExists2(absolutePath)) {
@@ -12083,14 +12111,17 @@ async function buildPromptInputs(testCase) {
       continue;
     }
     try {
-      const content = (await readFile3(absolutePath, "utf8")).replace(/\r\n/g, "\n");
-      guidelineContents.push(`=== ${path8.basename(absolutePath)} ===
-${content}`);
+      const content = (await readFile3(absolutePath, "utf8")).replace(/\r\n/g, "\n").trim();
+      guidelineParts.push({
+        content,
+        isFile: true,
+        displayPath: path8.basename(absolutePath)
+      });
     } catch (error) {
       logWarning(`Could not read guideline file ${absolutePath}: ${error.message}`);
     }
   }
-  const guidelines = guidelineContents.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
+  const guidelines = formatFileContents(guidelineParts);
   const segmentsByMessage = [];
   const fileContentsByPath = /* @__PURE__ */ new Map();
   for (const segment of testCase.input_segments) {
@@ -12292,6 +12323,20 @@ function cloneJsonValue(value) {
   }
   return cloneJsonObject(value);
 }
+function formatFileContents(parts) {
+  const fileCount = parts.filter((p) => p.isFile).length;
+  if (fileCount > 0) {
+    return parts.map((part) => {
+      if (part.isFile && part.displayPath) {
+        return `<file path="${part.displayPath}">
+${part.content}
+</file>`;
+      }
+      return part.content;
+    }).join("\n\n");
+  }
+  return parts.map((p) => p.content).join(" ");
+}
 async function resolveAssistantContent(content, searchRoots, verbose) {
   if (typeof content === "string") {
     return content;
@@ -12302,7 +12347,7 @@ async function resolveAssistantContent(content, searchRoots, verbose) {
   const parts = [];
   for (const entry of content) {
     if (typeof entry === "string") {
-      parts.push(entry);
+      parts.push({ content: entry, isFile: false });
       continue;
     }
     if (!isJsonObject(entry)) {
@@ -12324,8 +12369,8 @@ async function resolveAssistantContent(content, searchRoots, verbose) {
         continue;
       }
       try {
-        const fileContent = (await readFile3(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
-        parts.push(fileContent);
+        const fileContent = (await readFile3(resolvedPath, "utf8")).replace(/\r\n/g, "\n").trim();
+        parts.push({ content: fileContent, isFile: true, displayPath });
         if (verbose) {
           console.log(`  [Expected Assistant File] Found: ${displayPath}`);
           console.log(`    Resolved to: ${resolvedPath}`);
@@ -12337,21 +12382,21 @@ async function resolveAssistantContent(content, searchRoots, verbose) {
     }
     const textValue = asString(entry.text);
     if (typeof textValue === "string") {
-      parts.push(textValue);
+      parts.push({ content: textValue, isFile: false });
       continue;
     }
     const valueValue = asString(entry.value);
     if (typeof valueValue === "string") {
-      parts.push(valueValue);
+      parts.push({ content: valueValue, isFile: false });
       continue;
     }
-    parts.push(JSON.stringify(entry));
+    parts.push({ content: JSON.stringify(entry), isFile: false });
   }
-  return parts.join(" ");
+  return formatFileContents(parts);
 }
-async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
+async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId) {
   const execution = rawEvalCase.execution;
-  const candidateEvaluators = isJsonObject(execution) ? execution.evaluators ?? rawEvalCase.evaluators : rawEvalCase.evaluators;
+  const candidateEvaluators = isJsonObject(execution) ? execution.evaluators ?? rawEvalCase.evaluators : rawEvalCase.evaluators ?? globalExecution?.evaluators;
   if (candidateEvaluators === void 0) {
     return void 0;
   }
@@ -12389,6 +12434,8 @@ async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
             resolved.attempted.length > 0 ? resolved.attempted.map((attempt) => `  Tried: ${attempt}`) : void 0
           );
         }
+      } else {
+        resolvedCwd = searchRoots[0];
       }
       evaluators.push({
         name,
@@ -12417,8 +12464,7 @@ async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
       name,
       type: "llm_judge",
       prompt,
-      promptPath,
-      model
+      promptPath
     });
   }
   return evaluators.length > 0 ? evaluators : void 0;
@@ -14157,10 +14203,7 @@ var LlmJudgeEvaluator = class {
       prompt = substituteVariables(systemPrompt, variables);
       systemPrompt = buildSystemPrompt(hasReferenceAnswer);
     }
-    const metadata = {
-      ...systemPrompt !== void 0 ? { systemPrompt } : {},
-      ...context2.judgeModel !== void 0 ? { model: context2.judgeModel } : {}
-    };
+    const metadata = systemPrompt !== void 0 ? { systemPrompt } : {};
     const response = await judgeProvider.invoke({
       question: prompt,
       metadata,
@@ -14180,8 +14223,7 @@ var LlmJudgeEvaluator = class {
       provider: judgeProvider.id,
       prompt,
       target: context2.target.name,
-      ...systemPrompt !== void 0 ? { systemPrompt } : {},
-      ...context2.judgeModel !== void 0 ? { model: context2.judgeModel } : {}
+      ...systemPrompt !== void 0 && { systemPrompt }
     };
     return {
       score,
@@ -15164,8 +15206,7 @@ async function runLlmJudgeEvaluator(options) {
     now,
     judgeProvider,
     systemPrompt: customPrompt,
-    evaluator: config,
-    judgeModel: config.model
+    evaluator: config
   });
 }
 async function resolveCustomPrompt(config) {
@@ -16918,9 +16959,8 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
 
 // src/commands/eval/targets.ts
 import { constants as constants5 } from "node:fs";
-import { access as access5, readFile as readFile6 } from "node:fs/promises";
+import { access as access5 } from "node:fs/promises";
 import path13 from "node:path";
-import { parse as parse6 } from "yaml";
 var TARGET_FILE_CANDIDATES = [
   "targets.yaml",
   "targets.yml",
@@ -16942,18 +16982,8 @@ async function fileExists5(filePath) {
   }
 }
 async function readTestSuiteTarget(testFilePath) {
-  try {
-    const raw = await readFile6(path13.resolve(testFilePath), "utf8");
-    const parsed = parse6(raw);
-    if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
-      const targetValue = parsed.target;
-      if (typeof targetValue === "string" && targetValue.trim().length > 0) {
-        return targetValue.trim();
-      }
-    }
-  } catch {
-  }
-  return void 0;
+  const metadata = await readTestSuiteMetadata(testFilePath);
+  return metadata.target;
 }
 async function discoverTargetsFile(options) {
   const { explicitPath, testFilePath, repoRoot, cwd } = options;
@@ -17924,4 +17954,4 @@ export {
   createProgram,
   runCli
 };
-//# sourceMappingURL=chunk-J5HK75TC.js.map
+//# sourceMappingURL=chunk-7CJK3EYC.js.map