@agentv/core 0.22.2 → 0.25.0

package/dist/index.d.cts

@@ -1,5 +1,97 @@
 import * as ai from 'ai';
 
+/**
+ * Trace event types for capturing agent execution traces.
+ * Provides a normalized, provider-agnostic model for tool-call trajectories.
+ */
+/**
+ * Supported trace event types.
+ */
+type TraceEventType = 'model_step' | 'tool_call' | 'tool_result' | 'message' | 'error';
+/**
+ * Normalized trace event representing a single step in agent execution.
+ * Provider-agnostic format for tool-call trajectory evaluation.
+ */
+interface TraceEvent {
+    /** Event type */
+    readonly type: TraceEventType;
+    /** ISO 8601 timestamp */
+    readonly timestamp: string;
+    /** Stable identifier for pairing tool_call/tool_result */
+    readonly id?: string;
+    /** Tool name (for tool_call/tool_result) */
+    readonly name?: string;
+    /** Tool input - any JSON value */
+    readonly input?: unknown;
+    /** Tool output - any JSON value */
+    readonly output?: unknown;
+    /** Message content (for message/model_step) */
+    readonly text?: string;
+    /** Provider-specific metadata */
+    readonly metadata?: Record<string, unknown>;
+}
+/**
+ * Compact summary of a trace for lightweight persistence.
+ * Included in results by default to avoid payload bloat.
+ */
+interface TraceSummary {
+    /** Total number of events in trace */
+    readonly eventCount: number;
+    /** Unique tool names, sorted alphabetically */
+    readonly toolNames: readonly string[];
+    /** Map of tool name to call count */
+    readonly toolCallsByName: Readonly<Record<string, number>>;
+    /** Number of error events */
+    readonly errorCount: number;
+}
+/**
+ * Configuration for tool_trajectory evaluator.
+ */
+interface ToolTrajectoryEvaluatorConfig {
+    readonly name: string;
+    readonly type: 'tool_trajectory';
+    /** Matching mode */
+    readonly mode: 'any_order' | 'in_order' | 'exact';
+    /** Minimum call counts per tool (for any_order mode) */
+    readonly minimums?: Readonly<Record<string, number>>;
+    /** Expected tool sequence (for in_order/exact modes) */
+    readonly expected?: readonly ToolTrajectoryExpectedItem[];
+}
+/**
+ * Expected tool call item in a trajectory sequence.
+ */
+interface ToolTrajectoryExpectedItem {
+    readonly tool: string;
+}
+/**
+ * Expected tool call specification for expected_messages validation.
+ */
+interface ExpectedToolCall {
+    /** Tool name (required) */
+    readonly tool: string;
+    /** Tool input - if specified, must match exactly */
+    readonly input?: unknown;
+    /** Tool output - if specified, must match exactly */
+    readonly output?: unknown;
+}
+/**
+ * Type guard for TraceEventType values.
+ */
+declare function isTraceEventType(value: unknown): value is TraceEventType;
+/**
+ * Type guard for TraceEvent objects.
+ */
+declare function isTraceEvent(value: unknown): value is TraceEvent;
+/**
+ * Type guard for ExpectedToolCall objects.
+ */
+declare function isExpectedToolCall(value: unknown): value is ExpectedToolCall;
+/**
+ * Compute a lightweight summary from a full trace.
+ * Used for default result persistence without payload bloat.
+ */
+declare function computeTraceSummary(trace: readonly TraceEvent[]): TraceSummary;
+
 /**
  * JSON primitive values appearing in AgentV payloads.
  */
@@ -41,12 +133,21 @@ type UserTestMessage = {
     readonly role: 'user';
     readonly content: TestMessageContent;
 };
+/**
+ * Tool call specification for expected_messages validation.
+ */
+type TestMessageToolCall = {
+    readonly tool: string;
+    readonly input?: unknown;
+};
 /**
  * Assistant response message.
  */
 type AssistantTestMessage = {
     readonly role: 'assistant';
     readonly content: TestMessageContent;
+    /** Optional tool_calls for expected_messages validation against traces */
+    readonly tool_calls?: readonly TestMessageToolCall[];
 };
 /**
  * Tool invocation message.
@@ -75,7 +176,7 @@ declare function isJsonValue(value: unknown): value is JsonValue;
  * Guard validating raw test messages.
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge", "rubric"];
+declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite", "tool_trajectory", "expected_messages"];
 type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 type CodeEvaluatorConfig = {
@@ -99,7 +200,30 @@ type RubricItem = {
     readonly weight: number;
     readonly required: boolean;
 };
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
+type CompositeAggregatorConfig = {
+    readonly type: 'weighted_average';
+    readonly weights?: Record<string, number>;
+} | {
+    readonly type: 'code_judge';
+    readonly path: string;
+    readonly cwd?: string;
+} | {
+    readonly type: 'llm_judge';
+    readonly prompt?: string;
+    readonly promptPath?: string;
+    readonly model?: string;
+};
+type CompositeEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'composite';
+    readonly evaluators: readonly EvaluatorConfig[];
+    readonly aggregator: CompositeAggregatorConfig;
+};
+type ExpectedMessagesEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'expected_messages';
+};
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig | ToolTrajectoryEvaluatorConfig | ExpectedMessagesEvaluatorConfig;
 /**
  * Eval case definition sourced from AgentV specs.
  */
@@ -140,18 +264,24 @@ interface EvaluationResult {
     readonly evaluator_provider_request?: JsonObject;
     readonly evaluator_results?: readonly EvaluatorResult[];
     readonly error?: string;
+    /** Lightweight summary of the execution trace (always included when available) */
+    readonly trace_summary?: TraceSummary;
+    /** Full trace events (only included when --include-trace flag is set) */
+    readonly trace?: readonly TraceEvent[];
 }
 type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
 interface EvaluatorResult {
     readonly name: string;
     readonly type: EvaluatorKind;
     readonly score: number;
+    readonly weight?: number;
     readonly verdict?: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly reasoning?: string;
     readonly raw_request?: JsonObject;
     readonly evaluator_provider_request?: JsonObject;
+    readonly evaluator_results?: readonly EvaluatorResult[];
 }
 /**
  * Convenience accessor matching the Python hit_count property.
@@ -185,6 +315,10 @@ interface ProviderResponse {
     readonly reasoning?: string;
     readonly raw?: unknown;
     readonly usage?: JsonObject;
+    /** Normalized trace events from agent execution */
+    readonly trace?: readonly TraceEvent[];
+    /** Reference to external trace file (alternative to inline trace) */
+    readonly traceRef?: string;
 }
 interface Provider {
     readonly id: string;
@@ -338,6 +472,10 @@ declare function normalizeLineEndings(content: string): string;
  * This ensures consistent behavior across Windows (CRLF) and Unix (LF) systems.
  */
 declare function readTextFile(filePath: string): Promise<string>;
+/**
+ * Read a JSON file and parse it.
+ */
+declare function readJsonFile<T = unknown>(filePath: string): Promise<T>;
 /**
  * Find git repository root by walking up the directory tree.
  */
@@ -414,6 +552,8 @@ interface MockResolvedConfig {
     readonly delayMs?: number;
     readonly delayMinMs?: number;
     readonly delayMaxMs?: number;
+    /** Mock trace events for testing tool_trajectory evaluator */
+    readonly trace?: readonly TraceEvent[];
 }
 interface VSCodeResolvedConfig {
     readonly command: string;
@@ -490,7 +630,7 @@ type ResolvedTarget = {
     readonly providerBatching?: boolean;
     readonly config: CliResolvedConfig;
 };
-declare function resolveTargetDefinition(definition: TargetDefinition, env?: EnvLookup): ResolvedTarget;
+declare function resolveTargetDefinition(definition: TargetDefinition, env?: EnvLookup, evalFilePath?: string): ResolvedTarget;
 
 declare function readTargetDefinitions(filePath: string): Promise<readonly TargetDefinition[]>;
 declare function listTargetNames(definitions: readonly TargetDefinition[]): readonly string[];
@@ -542,6 +682,10 @@ interface EvaluationContext {
     readonly judgeProvider?: Provider;
     readonly evaluatorTemplateOverride?: string;
     readonly evaluator?: EvaluatorConfig;
+    /** Normalized trace events from provider execution (if available) */
+    readonly candidateTrace?: readonly TraceEvent[];
+    /** Lightweight summary of trace events (if available) */
+    readonly candidateTraceSummary?: TraceSummary;
 }
 interface EvaluationScore {
     readonly score: number;
@@ -552,6 +696,19 @@ interface EvaluationScore {
     readonly reasoning?: string;
     readonly rawAspects?: readonly string[];
     readonly evaluatorRawRequest?: JsonObject;
+    readonly evaluatorResults?: readonly ChildEvaluatorResult[];
+}
+interface ChildEvaluatorResult {
+    readonly name: string;
+    readonly type: string;
+    readonly score: number;
+    readonly weight?: number;
+    readonly verdict: EvaluationVerdict;
+    readonly hits: readonly string[];
+    readonly misses: readonly string[];
+    readonly reasoning?: string;
+    readonly evaluatorRawRequest?: JsonObject;
+    readonly evaluatorResults?: readonly ChildEvaluatorResult[];
 }
 interface Evaluator {
     readonly kind: string;
@@ -590,6 +747,50 @@ declare class CodeEvaluator implements Evaluator {
     constructor(options: CodeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
 }
+interface ToolTrajectoryEvaluatorOptions {
+    readonly config: ToolTrajectoryEvaluatorConfig;
+}
+declare class ToolTrajectoryEvaluator implements Evaluator {
+    readonly kind = "tool_trajectory";
+    private readonly config;
+    constructor(options: ToolTrajectoryEvaluatorOptions);
+    evaluate(context: EvaluationContext): EvaluationScore;
+    private evaluateAnyOrder;
+    private evaluateInOrder;
+    private evaluateExact;
+}
+/**
+ * Evaluator that validates tool_calls in expected_messages against the actual trace.
+ * Extracts tool_calls from assistant messages in expected_messages and compares them
+ * sequentially against tool_call events in the trace.
+ */
+declare class ExpectedMessagesEvaluator implements Evaluator {
+    readonly kind = "expected_messages";
+    evaluate(context: EvaluationContext): EvaluationScore;
+    private extractExpectedToolCalls;
+    private validateToolCalls;
+    private deepEquals;
+}
+interface EvaluatorFactory {
+    create(config: EvaluatorConfig, context: EvaluationContext): Evaluator;
+}
+interface CompositeEvaluatorOptions {
+    readonly config: CompositeEvaluatorConfig;
+    readonly evaluatorFactory: EvaluatorFactory;
+    readonly cwd?: string;
+}
+declare class CompositeEvaluator implements Evaluator {
+    readonly kind = "composite";
+    private readonly config;
+    private readonly evaluatorFactory;
+    private readonly cwd?;
+    constructor(options: CompositeEvaluatorOptions);
+    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
+    private aggregate;
+    private runWeightedAverage;
+    private runCodeAggregator;
+    private runLlmAggregator;
+}
 
 type MaybePromise<T> = T | Promise<T>;
 interface EvaluationCache {
@@ -660,4 +861,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, ExpectedMessagesEvaluator, type ExpectedMessagesEvaluatorConfig, type ExpectedToolCall, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type TestMessageToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceEvent, type TraceEventType, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isExpectedToolCall, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, isTraceEvent, isTraceEventType, listTargetNames, loadEvalCases, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.d.ts

@@ -1,5 +1,97 @@
 import * as ai from 'ai';
 
+/**
+ * Trace event types for capturing agent execution traces.
+ * Provides a normalized, provider-agnostic model for tool-call trajectories.
+ */
+/**
+ * Supported trace event types.
+ */
+type TraceEventType = 'model_step' | 'tool_call' | 'tool_result' | 'message' | 'error';
+/**
+ * Normalized trace event representing a single step in agent execution.
+ * Provider-agnostic format for tool-call trajectory evaluation.
+ */
+interface TraceEvent {
+    /** Event type */
+    readonly type: TraceEventType;
+    /** ISO 8601 timestamp */
+    readonly timestamp: string;
+    /** Stable identifier for pairing tool_call/tool_result */
+    readonly id?: string;
+    /** Tool name (for tool_call/tool_result) */
+    readonly name?: string;
+    /** Tool input - any JSON value */
+    readonly input?: unknown;
+    /** Tool output - any JSON value */
+    readonly output?: unknown;
+    /** Message content (for message/model_step) */
+    readonly text?: string;
+    /** Provider-specific metadata */
+    readonly metadata?: Record<string, unknown>;
+}
+/**
+ * Compact summary of a trace for lightweight persistence.
+ * Included in results by default to avoid payload bloat.
+ */
+interface TraceSummary {
+    /** Total number of events in trace */
+    readonly eventCount: number;
+    /** Unique tool names, sorted alphabetically */
+    readonly toolNames: readonly string[];
+    /** Map of tool name to call count */
+    readonly toolCallsByName: Readonly<Record<string, number>>;
+    /** Number of error events */
+    readonly errorCount: number;
+}
+/**
+ * Configuration for tool_trajectory evaluator.
+ */
+interface ToolTrajectoryEvaluatorConfig {
+    readonly name: string;
+    readonly type: 'tool_trajectory';
+    /** Matching mode */
+    readonly mode: 'any_order' | 'in_order' | 'exact';
+    /** Minimum call counts per tool (for any_order mode) */
+    readonly minimums?: Readonly<Record<string, number>>;
+    /** Expected tool sequence (for in_order/exact modes) */
+    readonly expected?: readonly ToolTrajectoryExpectedItem[];
+}
+/**
+ * Expected tool call item in a trajectory sequence.
+ */
+interface ToolTrajectoryExpectedItem {
+    readonly tool: string;
+}
+/**
+ * Expected tool call specification for expected_messages validation.
+ */
+interface ExpectedToolCall {
+    /** Tool name (required) */
+    readonly tool: string;
+    /** Tool input - if specified, must match exactly */
+    readonly input?: unknown;
+    /** Tool output - if specified, must match exactly */
+    readonly output?: unknown;
+}
+/**
+ * Type guard for TraceEventType values.
+ */
+declare function isTraceEventType(value: unknown): value is TraceEventType;
+/**
+ * Type guard for TraceEvent objects.
+ */
+declare function isTraceEvent(value: unknown): value is TraceEvent;
+/**
+ * Type guard for ExpectedToolCall objects.
+ */
+declare function isExpectedToolCall(value: unknown): value is ExpectedToolCall;
+/**
+ * Compute a lightweight summary from a full trace.
+ * Used for default result persistence without payload bloat.
+ */
+declare function computeTraceSummary(trace: readonly TraceEvent[]): TraceSummary;
+
 /**
  * JSON primitive values appearing in AgentV payloads.
  */
@@ -41,12 +133,21 @@ type UserTestMessage = {
     readonly role: 'user';
     readonly content: TestMessageContent;
 };
+/**
+ * Tool call specification for expected_messages validation.
+ */
+type TestMessageToolCall = {
+    readonly tool: string;
+    readonly input?: unknown;
+};
 /**
  * Assistant response message.
  */
 type AssistantTestMessage = {
     readonly role: 'assistant';
     readonly content: TestMessageContent;
+    /** Optional tool_calls for expected_messages validation against traces */
+    readonly tool_calls?: readonly TestMessageToolCall[];
 };
 /**
  * Tool invocation message.
@@ -75,7 +176,7 @@ declare function isJsonValue(value: unknown): value is JsonValue;
  * Guard validating raw test messages.
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge", "rubric"];
+declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite", "tool_trajectory", "expected_messages"];
 type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 type CodeEvaluatorConfig = {
@@ -99,7 +200,30 @@ type RubricItem = {
     readonly weight: number;
     readonly required: boolean;
 };
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
+type CompositeAggregatorConfig = {
+    readonly type: 'weighted_average';
+    readonly weights?: Record<string, number>;
+} | {
+    readonly type: 'code_judge';
+    readonly path: string;
+    readonly cwd?: string;
+} | {
+    readonly type: 'llm_judge';
+    readonly prompt?: string;
+    readonly promptPath?: string;
+    readonly model?: string;
+};
+type CompositeEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'composite';
+    readonly evaluators: readonly EvaluatorConfig[];
+    readonly aggregator: CompositeAggregatorConfig;
+};
+type ExpectedMessagesEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'expected_messages';
+};
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig | ToolTrajectoryEvaluatorConfig | ExpectedMessagesEvaluatorConfig;
 /**
  * Eval case definition sourced from AgentV specs.
  */
@@ -140,18 +264,24 @@ interface EvaluationResult {
     readonly evaluator_provider_request?: JsonObject;
     readonly evaluator_results?: readonly EvaluatorResult[];
     readonly error?: string;
+    /** Lightweight summary of the execution trace (always included when available) */
+    readonly trace_summary?: TraceSummary;
+    /** Full trace events (only included when --include-trace flag is set) */
+    readonly trace?: readonly TraceEvent[];
 }
 type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
 interface EvaluatorResult {
     readonly name: string;
     readonly type: EvaluatorKind;
     readonly score: number;
+    readonly weight?: number;
     readonly verdict?: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly reasoning?: string;
     readonly raw_request?: JsonObject;
     readonly evaluator_provider_request?: JsonObject;
+    readonly evaluator_results?: readonly EvaluatorResult[];
 }
 /**
  * Convenience accessor matching the Python hit_count property.
@@ -185,6 +315,10 @@ interface ProviderResponse {
     readonly reasoning?: string;
     readonly raw?: unknown;
     readonly usage?: JsonObject;
+    /** Normalized trace events from agent execution */
+    readonly trace?: readonly TraceEvent[];
+    /** Reference to external trace file (alternative to inline trace) */
+    readonly traceRef?: string;
 }
 interface Provider {
     readonly id: string;
@@ -338,6 +472,10 @@ declare function normalizeLineEndings(content: string): string;
  * This ensures consistent behavior across Windows (CRLF) and Unix (LF) systems.
  */
 declare function readTextFile(filePath: string): Promise<string>;
+/**
+ * Read a JSON file and parse it.
+ */
+declare function readJsonFile<T = unknown>(filePath: string): Promise<T>;
 /**
  * Find git repository root by walking up the directory tree.
  */
@@ -414,6 +552,8 @@ interface MockResolvedConfig {
     readonly delayMs?: number;
     readonly delayMinMs?: number;
     readonly delayMaxMs?: number;
+    /** Mock trace events for testing tool_trajectory evaluator */
+    readonly trace?: readonly TraceEvent[];
 }
 interface VSCodeResolvedConfig {
     readonly command: string;
@@ -490,7 +630,7 @@ type ResolvedTarget = {
     readonly providerBatching?: boolean;
     readonly config: CliResolvedConfig;
 };
-declare function resolveTargetDefinition(definition: TargetDefinition, env?: EnvLookup): ResolvedTarget;
+declare function resolveTargetDefinition(definition: TargetDefinition, env?: EnvLookup, evalFilePath?: string): ResolvedTarget;
 
 declare function readTargetDefinitions(filePath: string): Promise<readonly TargetDefinition[]>;
 declare function listTargetNames(definitions: readonly TargetDefinition[]): readonly string[];
@@ -542,6 +682,10 @@ interface EvaluationContext {
     readonly judgeProvider?: Provider;
     readonly evaluatorTemplateOverride?: string;
     readonly evaluator?: EvaluatorConfig;
+    /** Normalized trace events from provider execution (if available) */
+    readonly candidateTrace?: readonly TraceEvent[];
+    /** Lightweight summary of trace events (if available) */
+    readonly candidateTraceSummary?: TraceSummary;
 }
 interface EvaluationScore {
     readonly score: number;
@@ -552,6 +696,19 @@ interface EvaluationScore {
     readonly reasoning?: string;
     readonly rawAspects?: readonly string[];
     readonly evaluatorRawRequest?: JsonObject;
+    readonly evaluatorResults?: readonly ChildEvaluatorResult[];
+}
+interface ChildEvaluatorResult {
+    readonly name: string;
+    readonly type: string;
+    readonly score: number;
+    readonly weight?: number;
+    readonly verdict: EvaluationVerdict;
+    readonly hits: readonly string[];
+    readonly misses: readonly string[];
+    readonly reasoning?: string;
+    readonly evaluatorRawRequest?: JsonObject;
+    readonly evaluatorResults?: readonly ChildEvaluatorResult[];
 }
 interface Evaluator {
     readonly kind: string;
@@ -590,6 +747,50 @@ declare class CodeEvaluator implements Evaluator {
     constructor(options: CodeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
 }
+interface ToolTrajectoryEvaluatorOptions {
+    readonly config: ToolTrajectoryEvaluatorConfig;
+}
+declare class ToolTrajectoryEvaluator implements Evaluator {
+    readonly kind = "tool_trajectory";
+    private readonly config;
+    constructor(options: ToolTrajectoryEvaluatorOptions);
+    evaluate(context: EvaluationContext): EvaluationScore;
+    private evaluateAnyOrder;
+    private evaluateInOrder;
+    private evaluateExact;
+}
+/**
+ * Evaluator that validates tool_calls in expected_messages against the actual trace.
+ * Extracts tool_calls from assistant messages in expected_messages and compares them
+ * sequentially against tool_call events in the trace.
+ */
+declare class ExpectedMessagesEvaluator implements Evaluator {
+    readonly kind = "expected_messages";
+    evaluate(context: EvaluationContext): EvaluationScore;
+    private extractExpectedToolCalls;
+    private validateToolCalls;
+    private deepEquals;
+}
+interface EvaluatorFactory {
+    create(config: EvaluatorConfig, context: EvaluationContext): Evaluator;
+}
+interface CompositeEvaluatorOptions {
+    readonly config: CompositeEvaluatorConfig;
+    readonly evaluatorFactory: EvaluatorFactory;
+    readonly cwd?: string;
+}
+declare class CompositeEvaluator implements Evaluator {
+    readonly kind = "composite";
+    private readonly config;
+    private readonly evaluatorFactory;
+    private readonly cwd?;
+    constructor(options: CompositeEvaluatorOptions);
+    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
+    private aggregate;
+    private runWeightedAverage;
+    private runCodeAggregator;
+    private runLlmAggregator;
+}
 
 type MaybePromise<T> = T | Promise<T>;
 interface EvaluationCache {
@@ -660,4 +861,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, ExpectedMessagesEvaluator, type ExpectedMessagesEvaluatorConfig, type ExpectedToolCall, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type TestMessageToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceEvent, type TraceEventType, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isExpectedToolCall, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, isTraceEvent, isTraceEventType, listTargetNames, loadEvalCases, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };