@agentv/core 0.9.0 → 0.10.1

package/dist/index.d.cts

@@ -91,7 +91,6 @@ type LlmJudgeEvaluatorConfig = {
     readonly type: "llm_judge";
     readonly prompt?: string;
     readonly promptPath?: string;
-    readonly model?: string;
 };
 type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
 /**
@@ -102,9 +101,10 @@ interface EvalCase {
     readonly dataset?: string;
     readonly conversation_id?: string;
     readonly question: string;
+    readonly input_messages: readonly TestMessage[];
     readonly input_segments: readonly JsonObject[];
     readonly output_segments: readonly JsonObject[];
-    readonly reference_answer: string;
+    readonly reference_answer?: string;
     readonly guideline_paths: readonly string[];
     readonly guideline_patterns?: readonly string[];
     readonly file_paths: readonly string[];
@@ -129,7 +129,8 @@ interface EvaluationResult {
     readonly timestamp: string;
     readonly reasoning?: string;
     readonly raw_aspects?: readonly string[];
-    readonly raw_request?: JsonObject;
+    readonly agent_provider_request?: JsonObject;
+    readonly lm_provider_request?: JsonObject;
     readonly evaluator_raw_request?: JsonObject;
     readonly evaluator_results?: readonly EvaluatorResult[];
     readonly error?: string;
@@ -149,65 +150,6 @@ interface EvaluatorResult {
  */
 declare function getHitCount(result: Pick<EvaluationResult, "hits">): number;
 
-/**
- * Determine whether a path references guideline content (instructions or prompts).
- */
-declare function isGuidelineFile(filePath: string, patterns?: readonly string[]): boolean;
-/**
- * Extract fenced code blocks from AgentV user segments.
- */
-declare function extractCodeBlocks(segments: readonly JsonObject[]): readonly string[];
-type LoadOptions = {
-    readonly verbose?: boolean;
-    readonly evalId?: string;
-};
-/**
- * Load eval cases from a AgentV YAML specification file.
- */
-declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, options?: LoadOptions): Promise<readonly EvalCase[]>;
-/**
- * Build prompt inputs by consolidating user request context and guideline content.
- */
-declare function buildPromptInputs(testCase: EvalCase): Promise<{
-    question: string;
-    guidelines: string;
-    systemMessage?: string;
-}>;
-
-declare function fileExists(filePath: string): Promise<boolean>;
-/**
- * Normalize line endings to LF (\n).
- * This ensures consistent behavior across Windows (CRLF) and Unix (LF) systems.
- */
-declare function normalizeLineEndings(content: string): string;
-/**
- * Read a text file and normalize line endings to LF (\n).
- * This ensures consistent behavior across Windows (CRLF) and Unix (LF) systems.
- */
-declare function readTextFile(filePath: string): Promise<string>;
-/**
- * Find git repository root by walking up the directory tree.
- */
-declare function findGitRoot(startPath: string): Promise<string | null>;
-/**
- * Build a chain of directories walking from a file's location up to repo root.
- * Used for discovering configuration files like targets.yaml or config.yaml.
- */
-declare function buildDirectoryChain(filePath: string, repoRoot: string): readonly string[];
-/**
- * Build search roots for file resolution, matching yaml-parser behavior.
- * Searches from eval file directory up to repo root.
- */
-declare function buildSearchRoots(evalPath: string, repoRoot: string): readonly string[];
-/**
- * Resolve a file reference using search roots, matching yaml-parser behavior.
- */
-declare function resolveFileReference(rawValue: string, searchRoots: readonly string[]): Promise<{
-    readonly displayPath: string;
-    readonly resolvedPath?: string;
-    readonly attempted: readonly string[];
-}>;
-
 type ChatPrompt = AxChatRequest["chatPrompt"];
 type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
 interface ProviderRequest {
@@ -321,6 +263,74 @@ interface TargetDefinition {
     readonly retryStatusCodes?: unknown | undefined;
 }
 
+/**
+ * Read metadata from a test suite file (like target name).
+ * This is a convenience function for CLI tools that need metadata without loading all eval cases.
+ */
+declare function readTestSuiteMetadata(testFilePath: string): Promise<{
+    target?: string;
+}>;
+/**
+ * Determine whether a path references guideline content (instructions or prompts).
+ */
+declare function isGuidelineFile(filePath: string, patterns?: readonly string[]): boolean;
+/**
+ * Extract fenced code blocks from AgentV user segments.
+ */
+declare function extractCodeBlocks(segments: readonly JsonObject[]): readonly string[];
+type LoadOptions = {
+    readonly verbose?: boolean;
+    readonly evalId?: string;
+};
+/**
+ * Load eval cases from a AgentV YAML specification file.
+ */
+declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, options?: LoadOptions): Promise<readonly EvalCase[]>;
+/**
+ * Build prompt inputs by consolidating user request context and guideline content.
+ */
+interface PromptInputs {
+    readonly question: string;
+    readonly guidelines: string;
+    readonly chatPrompt?: ChatPrompt;
+    readonly systemMessage?: string;
+}
+declare function buildPromptInputs(testCase: EvalCase): Promise<PromptInputs>;
+
+declare function fileExists(filePath: string): Promise<boolean>;
+/**
+ * Normalize line endings to LF (\n).
+ * This ensures consistent behavior across Windows (CRLF) and Unix (LF) systems.
+ */
+declare function normalizeLineEndings(content: string): string;
+/**
+ * Read a text file and normalize line endings to LF (\n).
+ * This ensures consistent behavior across Windows (CRLF) and Unix (LF) systems.
+ */
+declare function readTextFile(filePath: string): Promise<string>;
+/**
+ * Find git repository root by walking up the directory tree.
+ */
+declare function findGitRoot(startPath: string): Promise<string | null>;
+/**
+ * Build a chain of directories walking from a file's location up to repo root.
+ * Used for discovering configuration files like targets.yaml or config.yaml.
+ */
+declare function buildDirectoryChain(filePath: string, repoRoot: string): readonly string[];
+/**
+ * Build search roots for file resolution, matching yaml-parser behavior.
+ * Searches from eval file directory up to repo root.
+ */
+declare function buildSearchRoots(evalPath: string, repoRoot: string): readonly string[];
+/**
+ * Resolve a file reference using search roots, matching yaml-parser behavior.
+ */
+declare function resolveFileReference(rawValue: string, searchRoots: readonly string[]): Promise<{
+    readonly displayPath: string;
+    readonly resolvedPath?: string;
+    readonly attempted: readonly string[];
+}>;
+
 interface RetryConfig {
     readonly maxRetries?: number;
     readonly initialDelayMs?: number;
@@ -486,12 +496,12 @@ interface EvaluationContext {
         readonly question: string;
         readonly guidelines: string;
         readonly systemMessage?: string;
+        readonly chatPrompt?: ChatPrompt;
     };
     readonly now: Date;
     readonly judgeProvider?: Provider;
     readonly systemPrompt?: string;
     readonly evaluator?: EvaluatorConfig;
-    readonly judgeModel?: string;
 }
 interface EvaluationScore {
     readonly score: number;
@@ -594,4 +604,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.d.ts

@@ -91,7 +91,6 @@ type LlmJudgeEvaluatorConfig = {
     readonly type: "llm_judge";
     readonly prompt?: string;
     readonly promptPath?: string;
-    readonly model?: string;
 };
 type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
 /**
@@ -102,9 +101,10 @@ interface EvalCase {
     readonly dataset?: string;
     readonly conversation_id?: string;
     readonly question: string;
+    readonly input_messages: readonly TestMessage[];
     readonly input_segments: readonly JsonObject[];
     readonly output_segments: readonly JsonObject[];
-    readonly reference_answer: string;
+    readonly reference_answer?: string;
     readonly guideline_paths: readonly string[];
     readonly guideline_patterns?: readonly string[];
     readonly file_paths: readonly string[];
@@ -129,7 +129,8 @@ interface EvaluationResult {
     readonly timestamp: string;
     readonly reasoning?: string;
     readonly raw_aspects?: readonly string[];
-    readonly raw_request?: JsonObject;
+    readonly agent_provider_request?: JsonObject;
+    readonly lm_provider_request?: JsonObject;
     readonly evaluator_raw_request?: JsonObject;
     readonly evaluator_results?: readonly EvaluatorResult[];
     readonly error?: string;
@@ -149,65 +150,6 @@ interface EvaluatorResult {
  */
 declare function getHitCount(result: Pick<EvaluationResult, "hits">): number;
 
-/**
- * Determine whether a path references guideline content (instructions or prompts).
- */
-declare function isGuidelineFile(filePath: string, patterns?: readonly string[]): boolean;
-/**
- * Extract fenced code blocks from AgentV user segments.
- */
-declare function extractCodeBlocks(segments: readonly JsonObject[]): readonly string[];
-type LoadOptions = {
-    readonly verbose?: boolean;
-    readonly evalId?: string;
-};
-/**
- * Load eval cases from a AgentV YAML specification file.
- */
-declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, options?: LoadOptions): Promise<readonly EvalCase[]>;
-/**
- * Build prompt inputs by consolidating user request context and guideline content.
- */
-declare function buildPromptInputs(testCase: EvalCase): Promise<{
-    question: string;
-    guidelines: string;
-    systemMessage?: string;
-}>;
-
-declare function fileExists(filePath: string): Promise<boolean>;
-/**
- * Normalize line endings to LF (\n).
- * This ensures consistent behavior across Windows (CRLF) and Unix (LF) systems.
- */
-declare function normalizeLineEndings(content: string): string;
-/**
- * Read a text file and normalize line endings to LF (\n).
- * This ensures consistent behavior across Windows (CRLF) and Unix (LF) systems.
- */
-declare function readTextFile(filePath: string): Promise<string>;
-/**
- * Find git repository root by walking up the directory tree.
- */
-declare function findGitRoot(startPath: string): Promise<string | null>;
-/**
- * Build a chain of directories walking from a file's location up to repo root.
- * Used for discovering configuration files like targets.yaml or config.yaml.
- */
-declare function buildDirectoryChain(filePath: string, repoRoot: string): readonly string[];
-/**
- * Build search roots for file resolution, matching yaml-parser behavior.
- * Searches from eval file directory up to repo root.
- */
-declare function buildSearchRoots(evalPath: string, repoRoot: string): readonly string[];
-/**
- * Resolve a file reference using search roots, matching yaml-parser behavior.
- */
-declare function resolveFileReference(rawValue: string, searchRoots: readonly string[]): Promise<{
-    readonly displayPath: string;
-    readonly resolvedPath?: string;
-    readonly attempted: readonly string[];
-}>;
-
 type ChatPrompt = AxChatRequest["chatPrompt"];
 type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
 interface ProviderRequest {
@@ -321,6 +263,74 @@ interface TargetDefinition {
     readonly retryStatusCodes?: unknown | undefined;
 }
 
+/**
+ * Read metadata from a test suite file (like target name).
+ * This is a convenience function for CLI tools that need metadata without loading all eval cases.
+ */
+declare function readTestSuiteMetadata(testFilePath: string): Promise<{
+    target?: string;
+}>;
+/**
+ * Determine whether a path references guideline content (instructions or prompts).
+ */
+declare function isGuidelineFile(filePath: string, patterns?: readonly string[]): boolean;
+/**
+ * Extract fenced code blocks from AgentV user segments.
+ */
+declare function extractCodeBlocks(segments: readonly JsonObject[]): readonly string[];
+type LoadOptions = {
+    readonly verbose?: boolean;
+    readonly evalId?: string;
+};
+/**
+ * Load eval cases from a AgentV YAML specification file.
+ */
+declare function loadEvalCases(evalFilePath: string, repoRoot: URL | string, options?: LoadOptions): Promise<readonly EvalCase[]>;
+/**
+ * Build prompt inputs by consolidating user request context and guideline content.
+ */
+interface PromptInputs {
+    readonly question: string;
+    readonly guidelines: string;
+    readonly chatPrompt?: ChatPrompt;
+    readonly systemMessage?: string;
+}
+declare function buildPromptInputs(testCase: EvalCase): Promise<PromptInputs>;
+
+declare function fileExists(filePath: string): Promise<boolean>;
+/**
+ * Normalize line endings to LF (\n).
+ * This ensures consistent behavior across Windows (CRLF) and Unix (LF) systems.
+ */
+declare function normalizeLineEndings(content: string): string;
+/**
+ * Read a text file and normalize line endings to LF (\n).
+ * This ensures consistent behavior across Windows (CRLF) and Unix (LF) systems.
+ */
+declare function readTextFile(filePath: string): Promise<string>;
+/**
+ * Find git repository root by walking up the directory tree.
+ */
+declare function findGitRoot(startPath: string): Promise<string | null>;
+/**
+ * Build a chain of directories walking from a file's location up to repo root.
+ * Used for discovering configuration files like targets.yaml or config.yaml.
+ */
+declare function buildDirectoryChain(filePath: string, repoRoot: string): readonly string[];
+/**
+ * Build search roots for file resolution, matching yaml-parser behavior.
+ * Searches from eval file directory up to repo root.
+ */
+declare function buildSearchRoots(evalPath: string, repoRoot: string): readonly string[];
+/**
+ * Resolve a file reference using search roots, matching yaml-parser behavior.
+ */
+declare function resolveFileReference(rawValue: string, searchRoots: readonly string[]): Promise<{
+    readonly displayPath: string;
+    readonly resolvedPath?: string;
+    readonly attempted: readonly string[];
+}>;
+
 interface RetryConfig {
     readonly maxRetries?: number;
     readonly initialDelayMs?: number;
@@ -486,12 +496,12 @@ interface EvaluationContext {
         readonly question: string;
         readonly guidelines: string;
         readonly systemMessage?: string;
+        readonly chatPrompt?: ChatPrompt;
     };
     readonly now: Date;
     readonly judgeProvider?: Provider;
     readonly systemPrompt?: string;
     readonly evaluator?: EvaluatorConfig;
-    readonly judgeModel?: string;
 }
 interface EvaluationScore {
     readonly score: number;
@@ -594,4 +604,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };