@agentv/core 0.10.0 → 0.11.0

package/dist/index.cjs

@@ -54,6 +54,7 @@ __export(index_exports, {
   loadEvalCases: () => loadEvalCases,
   normalizeLineEndings: () => normalizeLineEndings,
   readTargetDefinitions: () => readTargetDefinitions,
+  readTestSuiteMetadata: () => readTestSuiteMetadata,
   readTextFile: () => readTextFile,
   resolveAndCreateProvider: () => resolveAndCreateProvider,
   resolveFileReference: () => resolveFileReference,
@@ -239,6 +240,33 @@ var ANSI_YELLOW = "\x1B[33m";
 var ANSI_RESET = "\x1B[0m";
 var SCHEMA_EVAL_V2 = "agentv-eval-v2";
 var SCHEMA_CONFIG_V2 = "agentv-config-v2";
+async function readTestSuiteMetadata(testFilePath) {
+  try {
+    const absolutePath = import_node_path2.default.resolve(testFilePath);
+    const content = await (0, import_promises2.readFile)(absolutePath, "utf8");
+    const parsed = (0, import_yaml.parse)(content);
+    if (!isJsonObject(parsed)) {
+      return {};
+    }
+    return { target: extractTargetFromSuite(parsed) };
+  } catch {
+    return {};
+  }
+}
+function extractTargetFromSuite(suite) {
+  const execution = suite.execution;
+  if (execution && typeof execution === "object" && !Array.isArray(execution)) {
+    const executionTarget = execution.target;
+    if (typeof executionTarget === "string" && executionTarget.trim().length > 0) {
+      return executionTarget.trim();
+    }
+  }
+  const targetValue = suite.target;
+  if (typeof targetValue === "string" && targetValue.trim().length > 0) {
+    return targetValue.trim();
+  }
+  return void 0;
+}
 async function loadConfig(evalFilePath, repoRoot) {
   const directories = buildDirectoryChain(evalFilePath, repoRoot);
   for (const directory of directories) {
@@ -415,6 +443,8 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
     throw new Error(`Invalid test file format: ${evalFilePath} - missing 'evalcases' field`);
   }
   const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm_judge";
+  const globalExecution = isJsonObject(suite.execution) ? suite.execution : void 0;
+  const globalTarget = asString(globalExecution?.target) ?? asString(suite.target);
   const results = [];
   for (const rawEvalcase of rawTestcases) {
     if (!isJsonObject(rawEvalcase)) {
@@ -469,7 +499,7 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
     const referenceAnswer = expectedContent ? await resolveAssistantContent(expectedContent, searchRoots, verbose) : "";
     const question = inputTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
     const evalCaseEvaluatorKind = coerceEvaluator(evalcase.evaluator, id) ?? globalEvaluator;
-    const evaluators = await parseEvaluators(evalcase, searchRoots, id ?? "unknown");
+    const evaluators = await parseEvaluators(evalcase, globalExecution, searchRoots, id ?? "unknown");
     const userFilePaths = [];
     for (const segment of inputSegments) {
       if (segment.type === "file" && typeof segment.resolvedPath === "string") {
@@ -555,14 +585,13 @@ function formatSegment(segment) {
     const text = asString(segment.text);
     const filePath = asString(segment.path);
     if (text && filePath) {
-      return `=== ${filePath} ===
-${text}`;
+      return formatFileContents([{ content: text.trim(), isFile: true, displayPath: filePath }]);
     }
   }
   return void 0;
 }
 async function buildPromptInputs(testCase) {
-  const guidelineContents = [];
+  const guidelineParts = [];
   for (const rawPath of testCase.guideline_paths) {
     const absolutePath = import_node_path2.default.resolve(rawPath);
     if (!await fileExists2(absolutePath)) {
@@ -570,14 +599,17 @@ async function buildPromptInputs(testCase) {
       continue;
     }
     try {
-      const content = (await (0, import_promises2.readFile)(absolutePath, "utf8")).replace(/\r\n/g, "\n");
-      guidelineContents.push(`=== ${import_node_path2.default.basename(absolutePath)} ===
-${content}`);
+      const content = (await (0, import_promises2.readFile)(absolutePath, "utf8")).replace(/\r\n/g, "\n").trim();
+      guidelineParts.push({
+        content,
+        isFile: true,
+        displayPath: import_node_path2.default.basename(absolutePath)
+      });
     } catch (error) {
       logWarning(`Could not read guideline file ${absolutePath}: ${error.message}`);
     }
   }
-  const guidelines = guidelineContents.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
+  const guidelines = formatFileContents(guidelineParts);
   const segmentsByMessage = [];
   const fileContentsByPath = /* @__PURE__ */ new Map();
   for (const segment of testCase.input_segments) {
@@ -779,6 +811,20 @@ function cloneJsonValue(value) {
   }
   return cloneJsonObject(value);
 }
+function formatFileContents(parts) {
+  const fileCount = parts.filter((p) => p.isFile).length;
+  if (fileCount > 0) {
+    return parts.map((part) => {
+      if (part.isFile && part.displayPath) {
+        return `<file path="${part.displayPath}">
+${part.content}
+</file>`;
+      }
+      return part.content;
+    }).join("\n\n");
+  }
+  return parts.map((p) => p.content).join(" ");
+}
 async function resolveAssistantContent(content, searchRoots, verbose) {
   if (typeof content === "string") {
     return content;
@@ -789,7 +835,7 @@ async function resolveAssistantContent(content, searchRoots, verbose) {
   const parts = [];
   for (const entry of content) {
     if (typeof entry === "string") {
-      parts.push(entry);
+      parts.push({ content: entry, isFile: false });
       continue;
     }
     if (!isJsonObject(entry)) {
@@ -811,8 +857,8 @@ async function resolveAssistantContent(content, searchRoots, verbose) {
         continue;
       }
       try {
-        const fileContent = (await (0, import_promises2.readFile)(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
-        parts.push(fileContent);
+        const fileContent = (await (0, import_promises2.readFile)(resolvedPath, "utf8")).replace(/\r\n/g, "\n").trim();
+        parts.push({ content: fileContent, isFile: true, displayPath });
         if (verbose) {
           console.log(`  [Expected Assistant File] Found: ${displayPath}`);
           console.log(`    Resolved to: ${resolvedPath}`);
@@ -824,21 +870,21 @@ async function resolveAssistantContent(content, searchRoots, verbose) {
     }
     const textValue = asString(entry.text);
     if (typeof textValue === "string") {
-      parts.push(textValue);
+      parts.push({ content: textValue, isFile: false });
       continue;
     }
     const valueValue = asString(entry.value);
     if (typeof valueValue === "string") {
-      parts.push(valueValue);
+      parts.push({ content: valueValue, isFile: false });
       continue;
     }
-    parts.push(JSON.stringify(entry));
+    parts.push({ content: JSON.stringify(entry), isFile: false });
   }
-  return parts.join(" ");
+  return formatFileContents(parts);
 }
-async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
+async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId) {
   const execution = rawEvalCase.execution;
-  const candidateEvaluators = isJsonObject(execution) ? execution.evaluators ?? rawEvalCase.evaluators : rawEvalCase.evaluators;
+  const candidateEvaluators = isJsonObject(execution) ? execution.evaluators ?? rawEvalCase.evaluators : rawEvalCase.evaluators ?? globalExecution?.evaluators;
   if (candidateEvaluators === void 0) {
     return void 0;
   }
@@ -876,6 +922,8 @@ async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
             resolved.attempted.length > 0 ? resolved.attempted.map((attempt) => `  Tried: ${attempt}`) : void 0
           );
         }
+      } else {
+        resolvedCwd = searchRoots[0];
       }
       evaluators.push({
         name,
@@ -904,8 +952,7 @@ async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
       name,
       type: "llm_judge",
       prompt,
-      promptPath,
-      model
+      promptPath
     });
   }
   return evaluators.length > 0 ? evaluators : void 0;
@@ -3222,10 +3269,7 @@ var LlmJudgeEvaluator = class {
       prompt = substituteVariables(systemPrompt, variables);
       systemPrompt = buildSystemPrompt(hasReferenceAnswer);
     }
-    const metadata = {
-      ...systemPrompt !== void 0 ? { systemPrompt } : {},
-      ...context.judgeModel !== void 0 ? { model: context.judgeModel } : {}
-    };
+    const metadata = systemPrompt !== void 0 ? { systemPrompt } : {};
     const response = await judgeProvider.invoke({
       question: prompt,
       metadata,
@@ -3245,8 +3289,7 @@ var LlmJudgeEvaluator = class {
       provider: judgeProvider.id,
       prompt,
       target: context.target.name,
-      ...systemPrompt !== void 0 ? { systemPrompt } : {},
-      ...context.judgeModel !== void 0 ? { model: context.judgeModel } : {}
+      ...systemPrompt !== void 0 && { systemPrompt }
     };
     return {
       score,
@@ -4240,8 +4283,7 @@ async function runLlmJudgeEvaluator(options) {
     now,
     judgeProvider,
     systemPrompt: customPrompt,
-    evaluator: config,
-    judgeModel: config.model
+    evaluator: config
   });
 }
 async function resolveCustomPrompt(config) {
@@ -4427,6 +4469,7 @@ function createAgentKernel() {
   loadEvalCases,
   normalizeLineEndings,
   readTargetDefinitions,
+  readTestSuiteMetadata,
   readTextFile,
   resolveAndCreateProvider,
   resolveFileReference,