@agentv/core 0.10.0 → 0.11.0

package/dist/index.d.cts

@@ -91,7 +91,6 @@ type LlmJudgeEvaluatorConfig = {
     readonly type: "llm_judge";
     readonly prompt?: string;
     readonly promptPath?: string;
-    readonly model?: string;
 };
 type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
 /**
@@ -264,6 +263,13 @@ interface TargetDefinition {
     readonly retryStatusCodes?: unknown | undefined;
 }
 
+/**
+ * Read metadata from a test suite file (like target name).
+ * This is a convenience function for CLI tools that need metadata without loading all eval cases.
+ */
+declare function readTestSuiteMetadata(testFilePath: string): Promise<{
+    target?: string;
+}>;
 /**
  * Determine whether a path references guideline content (instructions or prompts).
  */
@@ -496,7 +502,6 @@ interface EvaluationContext {
     readonly judgeProvider?: Provider;
     readonly systemPrompt?: string;
     readonly evaluator?: EvaluatorConfig;
-    readonly judgeModel?: string;
 }
 interface EvaluationScore {
     readonly score: number;
@@ -599,4 +604,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.d.ts

@@ -91,7 +91,6 @@ type LlmJudgeEvaluatorConfig = {
     readonly type: "llm_judge";
     readonly prompt?: string;
     readonly promptPath?: string;
-    readonly model?: string;
 };
 type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
 /**
@@ -264,6 +263,13 @@ interface TargetDefinition {
     readonly retryStatusCodes?: unknown | undefined;
 }
 
+/**
+ * Read metadata from a test suite file (like target name).
+ * This is a convenience function for CLI tools that need metadata without loading all eval cases.
+ */
+declare function readTestSuiteMetadata(testFilePath: string): Promise<{
+    target?: string;
+}>;
 /**
  * Determine whether a path references guideline content (instructions or prompts).
  */
@@ -496,7 +502,6 @@ interface EvaluationContext {
     readonly judgeProvider?: Provider;
     readonly systemPrompt?: string;
     readonly evaluator?: EvaluatorConfig;
-    readonly judgeModel?: string;
 }
 interface EvaluationScore {
     readonly score: number;
@@ -599,4 +604,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.js

@@ -73,6 +73,33 @@ var ANSI_YELLOW = "\x1B[33m";
 var ANSI_RESET = "\x1B[0m";
 var SCHEMA_EVAL_V2 = "agentv-eval-v2";
 var SCHEMA_CONFIG_V2 = "agentv-config-v2";
+async function readTestSuiteMetadata(testFilePath) {
+  try {
+    const absolutePath = path.resolve(testFilePath);
+    const content = await readFile(absolutePath, "utf8");
+    const parsed = parse(content);
+    if (!isJsonObject(parsed)) {
+      return {};
+    }
+    return { target: extractTargetFromSuite(parsed) };
+  } catch {
+    return {};
+  }
+}
+function extractTargetFromSuite(suite) {
+  const execution = suite.execution;
+  if (execution && typeof execution === "object" && !Array.isArray(execution)) {
+    const executionTarget = execution.target;
+    if (typeof executionTarget === "string" && executionTarget.trim().length > 0) {
+      return executionTarget.trim();
+    }
+  }
+  const targetValue = suite.target;
+  if (typeof targetValue === "string" && targetValue.trim().length > 0) {
+    return targetValue.trim();
+  }
+  return void 0;
+}
 async function loadConfig(evalFilePath, repoRoot) {
   const directories = buildDirectoryChain(evalFilePath, repoRoot);
   for (const directory of directories) {
@@ -249,6 +276,8 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
     throw new Error(`Invalid test file format: ${evalFilePath} - missing 'evalcases' field`);
   }
   const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm_judge";
+  const globalExecution = isJsonObject(suite.execution) ? suite.execution : void 0;
+  const globalTarget = asString(globalExecution?.target) ?? asString(suite.target);
   const results = [];
   for (const rawEvalcase of rawTestcases) {
     if (!isJsonObject(rawEvalcase)) {
@@ -303,7 +332,7 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
     const referenceAnswer = expectedContent ? await resolveAssistantContent(expectedContent, searchRoots, verbose) : "";
     const question = inputTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
     const evalCaseEvaluatorKind = coerceEvaluator(evalcase.evaluator, id) ?? globalEvaluator;
-    const evaluators = await parseEvaluators(evalcase, searchRoots, id ?? "unknown");
+    const evaluators = await parseEvaluators(evalcase, globalExecution, searchRoots, id ?? "unknown");
     const userFilePaths = [];
     for (const segment of inputSegments) {
       if (segment.type === "file" && typeof segment.resolvedPath === "string") {
@@ -389,14 +418,13 @@ function formatSegment(segment) {
     const text = asString(segment.text);
     const filePath = asString(segment.path);
     if (text && filePath) {
-      return `=== ${filePath} ===
-${text}`;
+      return formatFileContents([{ content: text.trim(), isFile: true, displayPath: filePath }]);
     }
   }
   return void 0;
 }
 async function buildPromptInputs(testCase) {
-  const guidelineContents = [];
+  const guidelineParts = [];
   for (const rawPath of testCase.guideline_paths) {
     const absolutePath = path.resolve(rawPath);
     if (!await fileExists2(absolutePath)) {
@@ -404,14 +432,17 @@ async function buildPromptInputs(testCase) {
       continue;
     }
     try {
-      const content = (await readFile(absolutePath, "utf8")).replace(/\r\n/g, "\n");
-      guidelineContents.push(`=== ${path.basename(absolutePath)} ===
-${content}`);
+      const content = (await readFile(absolutePath, "utf8")).replace(/\r\n/g, "\n").trim();
+      guidelineParts.push({
+        content,
+        isFile: true,
+        displayPath: path.basename(absolutePath)
+      });
     } catch (error) {
       logWarning(`Could not read guideline file ${absolutePath}: ${error.message}`);
     }
   }
-  const guidelines = guidelineContents.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
+  const guidelines = formatFileContents(guidelineParts);
   const segmentsByMessage = [];
   const fileContentsByPath = /* @__PURE__ */ new Map();
   for (const segment of testCase.input_segments) {
@@ -613,6 +644,20 @@ function cloneJsonValue(value) {
   }
   return cloneJsonObject(value);
 }
+function formatFileContents(parts) {
+  const fileCount = parts.filter((p) => p.isFile).length;
+  if (fileCount > 0) {
+    return parts.map((part) => {
+      if (part.isFile && part.displayPath) {
+        return `<file path="${part.displayPath}">
+${part.content}
+</file>`;
+      }
+      return part.content;
+    }).join("\n\n");
+  }
+  return parts.map((p) => p.content).join(" ");
+}
 async function resolveAssistantContent(content, searchRoots, verbose) {
   if (typeof content === "string") {
     return content;
@@ -623,7 +668,7 @@ async function resolveAssistantContent(content, searchRoots, verbose) {
   const parts = [];
   for (const entry of content) {
     if (typeof entry === "string") {
-      parts.push(entry);
+      parts.push({ content: entry, isFile: false });
       continue;
     }
     if (!isJsonObject(entry)) {
@@ -645,8 +690,8 @@ async function resolveAssistantContent(content, searchRoots, verbose) {
         continue;
       }
       try {
-        const fileContent = (await readFile(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
-        parts.push(fileContent);
+        const fileContent = (await readFile(resolvedPath, "utf8")).replace(/\r\n/g, "\n").trim();
+        parts.push({ content: fileContent, isFile: true, displayPath });
         if (verbose) {
           console.log(`  [Expected Assistant File] Found: ${displayPath}`);
           console.log(`    Resolved to: ${resolvedPath}`);
@@ -658,21 +703,21 @@ async function resolveAssistantContent(content, searchRoots, verbose) {
     }
     const textValue = asString(entry.text);
     if (typeof textValue === "string") {
-      parts.push(textValue);
+      parts.push({ content: textValue, isFile: false });
       continue;
     }
     const valueValue = asString(entry.value);
     if (typeof valueValue === "string") {
-      parts.push(valueValue);
+      parts.push({ content: valueValue, isFile: false });
       continue;
     }
-    parts.push(JSON.stringify(entry));
+    parts.push({ content: JSON.stringify(entry), isFile: false });
   }
-  return parts.join(" ");
+  return formatFileContents(parts);
 }
-async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
+async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId) {
   const execution = rawEvalCase.execution;
-  const candidateEvaluators = isJsonObject(execution) ? execution.evaluators ?? rawEvalCase.evaluators : rawEvalCase.evaluators;
+  const candidateEvaluators = isJsonObject(execution) ? execution.evaluators ?? rawEvalCase.evaluators : rawEvalCase.evaluators ?? globalExecution?.evaluators;
   if (candidateEvaluators === void 0) {
     return void 0;
   }
@@ -710,6 +755,8 @@ async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
             resolved.attempted.length > 0 ? resolved.attempted.map((attempt) => `  Tried: ${attempt}`) : void 0
           );
         }
+      } else {
+        resolvedCwd = searchRoots[0];
       }
       evaluators.push({
         name,
@@ -738,8 +785,7 @@ async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
       name,
       type: "llm_judge",
       prompt,
-      promptPath,
-      model
+      promptPath
     });
   }
   return evaluators.length > 0 ? evaluators : void 0;
@@ -2532,10 +2578,7 @@ var LlmJudgeEvaluator = class {
       prompt = substituteVariables(systemPrompt, variables);
       systemPrompt = buildSystemPrompt(hasReferenceAnswer);
     }
-    const metadata = {
-      ...systemPrompt !== void 0 ? { systemPrompt } : {},
-      ...context.judgeModel !== void 0 ? { model: context.judgeModel } : {}
-    };
+    const metadata = systemPrompt !== void 0 ? { systemPrompt } : {};
     const response = await judgeProvider.invoke({
       question: prompt,
       metadata,
@@ -2555,8 +2598,7 @@ var LlmJudgeEvaluator = class {
       provider: judgeProvider.id,
       prompt,
       target: context.target.name,
-      ...systemPrompt !== void 0 ? { systemPrompt } : {},
-      ...context.judgeModel !== void 0 ? { model: context.judgeModel } : {}
+      ...systemPrompt !== void 0 && { systemPrompt }
     };
     return {
       score,
@@ -3550,8 +3592,7 @@ async function runLlmJudgeEvaluator(options) {
     now,
     judgeProvider,
     systemPrompt: customPrompt,
-    evaluator: config,
-    judgeModel: config.model
+    evaluator: config
   });
 }
 async function resolveCustomPrompt(config) {
@@ -3736,6 +3777,7 @@ export {
   loadEvalCases,
   normalizeLineEndings,
   readTargetDefinitions,
+  readTestSuiteMetadata,
   readTextFile,
   resolveAndCreateProvider,
   resolveFileReference,