@agentv/core 0.10.0 → 0.10.1

package/dist/index.d.cts

@@ -91,7 +91,6 @@ type LlmJudgeEvaluatorConfig = {
     readonly type: "llm_judge";
     readonly prompt?: string;
     readonly promptPath?: string;
-    readonly model?: string;
 };
 type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
 /**
@@ -264,6 +263,13 @@ interface TargetDefinition {
     readonly retryStatusCodes?: unknown | undefined;
 }
 
+/**
+ * Read metadata from a test suite file (like target name).
+ * This is a convenience function for CLI tools that need metadata without loading all eval cases.
+ */
+declare function readTestSuiteMetadata(testFilePath: string): Promise<{
+    target?: string;
+}>;
 /**
  * Determine whether a path references guideline content (instructions or prompts).
  */
@@ -496,7 +502,6 @@ interface EvaluationContext {
     readonly judgeProvider?: Provider;
     readonly systemPrompt?: string;
     readonly evaluator?: EvaluatorConfig;
-    readonly judgeModel?: string;
 }
 interface EvaluationScore {
     readonly score: number;
@@ -599,4 +604,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.d.ts

@@ -91,7 +91,6 @@ type LlmJudgeEvaluatorConfig = {
     readonly type: "llm_judge";
     readonly prompt?: string;
     readonly promptPath?: string;
-    readonly model?: string;
 };
 type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
 /**
@@ -264,6 +263,13 @@ interface TargetDefinition {
     readonly retryStatusCodes?: unknown | undefined;
 }
 
+/**
+ * Read metadata from a test suite file (like target name).
+ * This is a convenience function for CLI tools that need metadata without loading all eval cases.
+ */
+declare function readTestSuiteMetadata(testFilePath: string): Promise<{
+    target?: string;
+}>;
 /**
  * Determine whether a path references guideline content (instructions or prompts).
  */
@@ -496,7 +502,6 @@ interface EvaluationContext {
     readonly judgeProvider?: Provider;
     readonly systemPrompt?: string;
     readonly evaluator?: EvaluatorConfig;
-    readonly judgeModel?: string;
 }
 interface EvaluationScore {
     readonly score: number;
@@ -599,4 +604,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.js

@@ -73,6 +73,33 @@ var ANSI_YELLOW = "\x1B[33m";
 var ANSI_RESET = "\x1B[0m";
 var SCHEMA_EVAL_V2 = "agentv-eval-v2";
 var SCHEMA_CONFIG_V2 = "agentv-config-v2";
+async function readTestSuiteMetadata(testFilePath) {
+  try {
+    const absolutePath = path.resolve(testFilePath);
+    const content = await readFile(absolutePath, "utf8");
+    const parsed = parse(content);
+    if (!isJsonObject(parsed)) {
+      return {};
+    }
+    return { target: extractTargetFromSuite(parsed) };
+  } catch {
+    return {};
+  }
+}
+function extractTargetFromSuite(suite) {
+  const execution = suite.execution;
+  if (execution && typeof execution === "object" && !Array.isArray(execution)) {
+    const executionTarget = execution.target;
+    if (typeof executionTarget === "string" && executionTarget.trim().length > 0) {
+      return executionTarget.trim();
+    }
+  }
+  const targetValue = suite.target;
+  if (typeof targetValue === "string" && targetValue.trim().length > 0) {
+    return targetValue.trim();
+  }
+  return void 0;
+}
 async function loadConfig(evalFilePath, repoRoot) {
   const directories = buildDirectoryChain(evalFilePath, repoRoot);
   for (const directory of directories) {
@@ -249,6 +276,8 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
     throw new Error(`Invalid test file format: ${evalFilePath} - missing 'evalcases' field`);
   }
   const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm_judge";
+  const globalExecution = isJsonObject(suite.execution) ? suite.execution : void 0;
+  const globalTarget = asString(globalExecution?.target) ?? asString(suite.target);
   const results = [];
   for (const rawEvalcase of rawTestcases) {
     if (!isJsonObject(rawEvalcase)) {
@@ -303,7 +332,7 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
     const referenceAnswer = expectedContent ? await resolveAssistantContent(expectedContent, searchRoots, verbose) : "";
     const question = inputTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
     const evalCaseEvaluatorKind = coerceEvaluator(evalcase.evaluator, id) ?? globalEvaluator;
-    const evaluators = await parseEvaluators(evalcase, searchRoots, id ?? "unknown");
+    const evaluators = await parseEvaluators(evalcase, globalExecution, searchRoots, id ?? "unknown");
     const userFilePaths = [];
     for (const segment of inputSegments) {
       if (segment.type === "file" && typeof segment.resolvedPath === "string") {
@@ -670,9 +699,9 @@ async function resolveAssistantContent(content, searchRoots, verbose) {
   }
   return parts.join(" ");
 }
-async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
+async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId) {
   const execution = rawEvalCase.execution;
-  const candidateEvaluators = isJsonObject(execution) ? execution.evaluators ?? rawEvalCase.evaluators : rawEvalCase.evaluators;
+  const candidateEvaluators = isJsonObject(execution) ? execution.evaluators ?? rawEvalCase.evaluators : rawEvalCase.evaluators ?? globalExecution?.evaluators;
   if (candidateEvaluators === void 0) {
     return void 0;
   }
@@ -710,6 +739,8 @@ async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
             resolved.attempted.length > 0 ? resolved.attempted.map((attempt) => `  Tried: ${attempt}`) : void 0
           );
         }
+      } else {
+        resolvedCwd = searchRoots[0];
       }
       evaluators.push({
         name,
@@ -738,8 +769,7 @@ async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
       name,
       type: "llm_judge",
       prompt,
-      promptPath,
-      model
+      promptPath
     });
   }
   return evaluators.length > 0 ? evaluators : void 0;
@@ -2532,10 +2562,7 @@ var LlmJudgeEvaluator = class {
       prompt = substituteVariables(systemPrompt, variables);
       systemPrompt = buildSystemPrompt(hasReferenceAnswer);
     }
-    const metadata = {
-      ...systemPrompt !== void 0 ? { systemPrompt } : {},
-      ...context.judgeModel !== void 0 ? { model: context.judgeModel } : {}
-    };
+    const metadata = systemPrompt !== void 0 ? { systemPrompt } : {};
     const response = await judgeProvider.invoke({
       question: prompt,
       metadata,
@@ -2555,8 +2582,7 @@ var LlmJudgeEvaluator = class {
       provider: judgeProvider.id,
       prompt,
       target: context.target.name,
-      ...systemPrompt !== void 0 ? { systemPrompt } : {},
-      ...context.judgeModel !== void 0 ? { model: context.judgeModel } : {}
+      ...systemPrompt !== void 0 && { systemPrompt }
     };
     return {
       score,
@@ -3550,8 +3576,7 @@ async function runLlmJudgeEvaluator(options) {
     now,
     judgeProvider,
     systemPrompt: customPrompt,
-    evaluator: config,
-    judgeModel: config.model
+    evaluator: config
   });
 }
 async function resolveCustomPrompt(config) {
@@ -3736,6 +3761,7 @@ export {
   loadEvalCases,
   normalizeLineEndings,
   readTargetDefinitions,
+  readTestSuiteMetadata,
   readTextFile,
   resolveAndCreateProvider,
   resolveFileReference,