@agentv/core 0.26.0 → 1.2.0

package/dist/index.d.cts

@@ -65,17 +65,6 @@ interface ToolTrajectoryEvaluatorConfig {
 interface ToolTrajectoryExpectedItem {
     readonly tool: string;
 }
-/**
- * Expected tool call specification for expected_messages validation.
- */
-interface ExpectedToolCall {
-    /** Tool name (required) */
-    readonly tool: string;
-    /** Tool input - if specified, must match exactly */
-    readonly input?: unknown;
-    /** Tool output - if specified, must match exactly */
-    readonly output?: unknown;
-}
 /**
  * Type guard for TraceEventType values.
  */
@@ -84,10 +73,6 @@ declare function isTraceEventType(value: unknown): value is TraceEventType;
  * Type guard for TraceEvent objects.
  */
 declare function isTraceEvent(value: unknown): value is TraceEvent;
-/**
- * Type guard for ExpectedToolCall objects.
- */
-declare function isExpectedToolCall(value: unknown): value is ExpectedToolCall;
 /**
  * Compute a lightweight summary from a full trace.
  * Used for default result persistence without payload bloat.
@@ -135,21 +120,12 @@ type UserTestMessage = {
     readonly role: 'user';
     readonly content: TestMessageContent;
 };
-/**
- * Tool call specification for expected_messages validation.
- */
-type TestMessageToolCall = {
-    readonly tool: string;
-    readonly input?: unknown;
-};
 /**
  * Assistant response message.
  */
 type AssistantTestMessage = {
     readonly role: 'assistant';
     readonly content: TestMessageContent;
-    /** Optional tool_calls for expected_messages validation against traces */
-    readonly tool_calls?: readonly TestMessageToolCall[];
 };
 /**
  * Tool invocation message.
@@ -176,9 +152,12 @@ declare function isJsonObject(value: unknown): value is JsonObject;
 declare function isJsonValue(value: unknown): value is JsonValue;
 /**
  * Guard validating raw test messages.
+ * A valid test message has:
+ * - A valid role (system, user, assistant, tool)
+ * - Either content (string or array of objects) OR tool_calls (for assistant messages)
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite", "tool_trajectory", "expected_messages"];
+declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite", "tool_trajectory"];
 type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 type CodeEvaluatorConfig = {
@@ -224,12 +203,7 @@ type CompositeEvaluatorConfig = {
     readonly aggregator: CompositeAggregatorConfig;
     readonly weight?: number;
 };
-type ExpectedMessagesEvaluatorConfig = {
-    readonly name: string;
-    readonly type: 'expected_messages';
-    readonly weight?: number;
-};
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig | ToolTrajectoryEvaluatorConfig | ExpectedMessagesEvaluatorConfig;
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig | ToolTrajectoryEvaluatorConfig;
 /**
  * Eval case definition sourced from AgentV specs.
  */
@@ -240,7 +214,7 @@ interface EvalCase {
     readonly question: string;
     readonly input_messages: readonly TestMessage[];
     readonly input_segments: readonly JsonObject[];
-    readonly expected_segments: readonly JsonObject[];
+    readonly expected_messages: readonly JsonObject[];
     readonly reference_answer?: string;
     readonly guideline_paths: readonly string[];
     readonly guideline_patterns?: readonly string[];
@@ -690,6 +664,8 @@ interface EvaluationContext {
     readonly evaluator?: EvaluatorConfig;
     /** Normalized trace events from provider execution (if available) */
     readonly candidateTrace?: readonly TraceEvent[];
+    /** File path to trace data (alternative to inline candidateTrace) */
+    readonly candidateTraceRef?: string;
     /** Lightweight summary of trace events (if available) */
     readonly candidateTraceSummary?: TraceSummary;
 }
@@ -765,18 +741,6 @@ declare class ToolTrajectoryEvaluator implements Evaluator {
     private evaluateInOrder;
     private evaluateExact;
 }
-/**
- * Evaluator that validates tool_calls in expected_messages against the actual trace.
- * Extracts tool_calls from assistant messages in expected_messages and compares them
- * sequentially against tool_call events in the trace.
- */
-declare class ExpectedMessagesEvaluator implements Evaluator {
-    readonly kind = "expected_messages";
-    evaluate(context: EvaluationContext): EvaluationScore;
-    private extractExpectedToolCalls;
-    private validateToolCalls;
-    private deepEquals;
-}
 interface EvaluatorFactory {
     create(config: EvaluatorConfig, context: EvaluationContext): Evaluator;
 }
@@ -867,4 +831,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, ExpectedMessagesEvaluator, type ExpectedMessagesEvaluatorConfig, type ExpectedToolCall, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type TestMessageToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceEvent, type TraceEventType, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isExpectedToolCall, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, isTraceEvent, isTraceEventType, listTargetNames, loadEvalCases, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceEvent, type TraceEventType, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, isTraceEvent, isTraceEventType, listTargetNames, loadEvalCases, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.d.ts

@@ -65,17 +65,6 @@ interface ToolTrajectoryEvaluatorConfig {
 interface ToolTrajectoryExpectedItem {
     readonly tool: string;
 }
-/**
- * Expected tool call specification for expected_messages validation.
- */
-interface ExpectedToolCall {
-    /** Tool name (required) */
-    readonly tool: string;
-    /** Tool input - if specified, must match exactly */
-    readonly input?: unknown;
-    /** Tool output - if specified, must match exactly */
-    readonly output?: unknown;
-}
 /**
  * Type guard for TraceEventType values.
  */
@@ -84,10 +73,6 @@ declare function isTraceEventType(value: unknown): value is TraceEventType;
  * Type guard for TraceEvent objects.
  */
 declare function isTraceEvent(value: unknown): value is TraceEvent;
-/**
- * Type guard for ExpectedToolCall objects.
- */
-declare function isExpectedToolCall(value: unknown): value is ExpectedToolCall;
 /**
  * Compute a lightweight summary from a full trace.
  * Used for default result persistence without payload bloat.
@@ -135,21 +120,12 @@ type UserTestMessage = {
     readonly role: 'user';
     readonly content: TestMessageContent;
 };
-/**
- * Tool call specification for expected_messages validation.
- */
-type TestMessageToolCall = {
-    readonly tool: string;
-    readonly input?: unknown;
-};
 /**
  * Assistant response message.
  */
 type AssistantTestMessage = {
     readonly role: 'assistant';
     readonly content: TestMessageContent;
-    /** Optional tool_calls for expected_messages validation against traces */
-    readonly tool_calls?: readonly TestMessageToolCall[];
 };
 /**
  * Tool invocation message.
@@ -176,9 +152,12 @@ declare function isJsonObject(value: unknown): value is JsonObject;
 declare function isJsonValue(value: unknown): value is JsonValue;
 /**
  * Guard validating raw test messages.
+ * A valid test message has:
+ * - A valid role (system, user, assistant, tool)
+ * - Either content (string or array of objects) OR tool_calls (for assistant messages)
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite", "tool_trajectory", "expected_messages"];
+declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite", "tool_trajectory"];
 type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 type CodeEvaluatorConfig = {
@@ -224,12 +203,7 @@ type CompositeEvaluatorConfig = {
     readonly aggregator: CompositeAggregatorConfig;
     readonly weight?: number;
 };
-type ExpectedMessagesEvaluatorConfig = {
-    readonly name: string;
-    readonly type: 'expected_messages';
-    readonly weight?: number;
-};
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig | ToolTrajectoryEvaluatorConfig | ExpectedMessagesEvaluatorConfig;
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig | ToolTrajectoryEvaluatorConfig;
 /**
  * Eval case definition sourced from AgentV specs.
  */
@@ -240,7 +214,7 @@ interface EvalCase {
     readonly question: string;
     readonly input_messages: readonly TestMessage[];
     readonly input_segments: readonly JsonObject[];
-    readonly expected_segments: readonly JsonObject[];
+    readonly expected_messages: readonly JsonObject[];
     readonly reference_answer?: string;
     readonly guideline_paths: readonly string[];
     readonly guideline_patterns?: readonly string[];
@@ -690,6 +664,8 @@ interface EvaluationContext {
     readonly evaluator?: EvaluatorConfig;
     /** Normalized trace events from provider execution (if available) */
     readonly candidateTrace?: readonly TraceEvent[];
+    /** File path to trace data (alternative to inline candidateTrace) */
+    readonly candidateTraceRef?: string;
     /** Lightweight summary of trace events (if available) */
     readonly candidateTraceSummary?: TraceSummary;
 }
@@ -765,18 +741,6 @@ declare class ToolTrajectoryEvaluator implements Evaluator {
     private evaluateInOrder;
     private evaluateExact;
 }
-/**
- * Evaluator that validates tool_calls in expected_messages against the actual trace.
- * Extracts tool_calls from assistant messages in expected_messages and compares them
- * sequentially against tool_call events in the trace.
- */
-declare class ExpectedMessagesEvaluator implements Evaluator {
-    readonly kind = "expected_messages";
-    evaluate(context: EvaluationContext): EvaluationScore;
-    private extractExpectedToolCalls;
-    private validateToolCalls;
-    private deepEquals;
-}
 interface EvaluatorFactory {
     create(config: EvaluatorConfig, context: EvaluationContext): Evaluator;
 }
@@ -867,4 +831,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, ExpectedMessagesEvaluator, type ExpectedMessagesEvaluatorConfig, type ExpectedToolCall, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type TestMessageToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceEvent, type TraceEventType, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isExpectedToolCall, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, isTraceEvent, isTraceEventType, listTargetNames, loadEvalCases, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceEvent, type TraceEventType, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, isTraceEvent, isTraceEventType, listTargetNames, loadEvalCases, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };