@agentv/core 1.2.0 → 1.4.0

package/dist/index.d.cts

@@ -4,32 +4,6 @@ import * as ai from 'ai';
  * Trace event types for capturing agent execution traces.
  * Provides a normalized, provider-agnostic model for tool-call trajectories.
  */
-/**
- * Supported trace event types.
- */
-type TraceEventType = 'model_step' | 'tool_call' | 'tool_result' | 'message' | 'error';
-/**
- * Normalized trace event representing a single step in agent execution.
- * Provider-agnostic format for tool-call trajectory evaluation.
- */
-interface TraceEvent {
-    /** Event type */
-    readonly type: TraceEventType;
-    /** ISO 8601 timestamp */
-    readonly timestamp: string;
-    /** Stable identifier for pairing tool_call/tool_result */
-    readonly id?: string;
-    /** Tool name (for tool_call/tool_result) */
-    readonly name?: string;
-    /** Tool input - any JSON value */
-    readonly input?: unknown;
-    /** Tool output - any JSON value */
-    readonly output?: unknown;
-    /** Message content (for message/model_step) */
-    readonly text?: string;
-    /** Provider-specific metadata */
-    readonly metadata?: Record<string, unknown>;
-}
 /**
  * Compact summary of a trace for lightweight persistence.
  * Included in results by default to avoid payload bloat.
@@ -66,18 +40,19 @@ interface ToolTrajectoryExpectedItem {
     readonly tool: string;
 }
 /**
- * Type guard for TraceEventType values.
+ * Simplified input type for computeTraceSummary.
+ * Matches OutputMessage structure without requiring full provider/types import.
  */
-declare function isTraceEventType(value: unknown): value is TraceEventType;
-/**
- * Type guard for TraceEvent objects.
- */
-declare function isTraceEvent(value: unknown): value is TraceEvent;
+interface OutputMessageLike {
+    readonly toolCalls?: readonly {
+        readonly tool: string;
+    }[];
+}
 /**
- * Compute a lightweight summary from a full trace.
+ * Compute a lightweight summary from output messages.
  * Used for default result persistence without payload bloat.
  */
-declare function computeTraceSummary(trace: readonly TraceEvent[]): TraceSummary;
+declare function computeTraceSummary(messages: readonly OutputMessageLike[]): TraceSummary;
 
 /**
  * JSON primitive values appearing in AgentV payloads.
@@ -105,7 +80,7 @@ type TestMessageRole = (typeof TEST_MESSAGE_ROLE_VALUES)[number];
 /**
  * Text or structured payload attached to a message.
  */
-type TestMessageContent = string | readonly JsonObject[];
+type TestMessageContent = string | JsonObject | readonly JsonObject[];
 /**
  * System-authored instruction message.
  */
@@ -246,8 +221,6 @@ interface EvaluationResult {
     readonly error?: string;
     /** Lightweight summary of the execution trace (always included when available) */
     readonly trace_summary?: TraceSummary;
-    /** Full trace events (only included when --include-trace flag is set) */
-    readonly trace?: readonly TraceEvent[];
 }
 type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
 interface EvaluatorResult {
@@ -290,15 +263,45 @@ interface ProviderRequest {
     readonly metadata?: JsonObject;
     readonly signal?: AbortSignal;
 }
+/**
+ * A tool call within an output message.
+ * Represents a single tool invocation with its input and optional output.
+ */
+interface ToolCall {
+    /** Tool name */
+    readonly tool: string;
+    /** Tool input arguments */
+    readonly input?: unknown;
+    /** Tool output result */
+    readonly output?: unknown;
+    /** Stable identifier for pairing tool calls */
+    readonly id?: string;
+    /** ISO 8601 timestamp */
+    readonly timestamp?: string;
+}
+/**
+ * An output message from agent execution.
+ * Represents a single message in the conversation with optional tool calls.
+ */
+interface OutputMessage {
+    /** Message role (e.g., 'assistant', 'user', 'tool') */
+    readonly role: string;
+    /** Optional name for the message sender */
+    readonly name?: string;
+    /** Message content */
+    readonly content?: unknown;
+    /** Tool calls made in this message */
+    readonly toolCalls?: readonly ToolCall[];
+    /** ISO 8601 timestamp */
+    readonly timestamp?: string;
+    /** Provider-specific metadata */
+    readonly metadata?: Record<string, unknown>;
+}
 interface ProviderResponse {
-    readonly text: string;
-    readonly reasoning?: string;
     readonly raw?: unknown;
     readonly usage?: JsonObject;
-    /** Normalized trace events from agent execution */
-    readonly trace?: readonly TraceEvent[];
-    /** Reference to external trace file (alternative to inline trace) */
-    readonly traceRef?: string;
+    /** Output messages from agent execution (primary source for tool trajectory) */
+    readonly outputMessages?: readonly OutputMessage[];
 }
 interface Provider {
     readonly id: string;
@@ -532,8 +535,6 @@ interface MockResolvedConfig {
     readonly delayMs?: number;
     readonly delayMinMs?: number;
     readonly delayMaxMs?: number;
-    /** Mock trace events for testing tool_trajectory evaluator */
-    readonly trace?: readonly TraceEvent[];
 }
 interface VSCodeResolvedConfig {
     readonly command: string;
@@ -559,6 +560,7 @@ interface CliResolvedConfig {
     readonly timeoutMs?: number;
     readonly healthcheck?: CliHealthcheck;
     readonly verbose?: boolean;
+    readonly keepTempFiles?: boolean;
 }
 type ResolvedTarget = {
     readonly kind: 'azure';
@@ -662,12 +664,10 @@ interface EvaluationContext {
     readonly judgeProvider?: Provider;
     readonly evaluatorTemplateOverride?: string;
     readonly evaluator?: EvaluatorConfig;
-    /** Normalized trace events from provider execution (if available) */
-    readonly candidateTrace?: readonly TraceEvent[];
-    /** File path to trace data (alternative to inline candidateTrace) */
-    readonly candidateTraceRef?: string;
+    /** Output messages from agent execution (primary source for tool trajectory) */
+    readonly outputMessages?: readonly OutputMessage[];
     /** Lightweight summary of trace events (if available) */
-    readonly candidateTraceSummary?: TraceSummary;
+    readonly traceSummary?: TraceSummary;
 }
 interface EvaluationScore {
     readonly score: number;
@@ -737,6 +737,14 @@ declare class ToolTrajectoryEvaluator implements Evaluator {
     private readonly config;
     constructor(options: ToolTrajectoryEvaluatorOptions);
     evaluate(context: EvaluationContext): EvaluationScore;
+    /**
+     * Extract tool calls from output messages.
+     */
+    private extractToolCallsFromMessages;
+    /**
+     * Build a summary from extracted tool calls.
+     */
+    private buildSummary;
     private evaluateAnyOrder;
     private evaluateInOrder;
     private evaluateExact;
@@ -831,4 +839,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceEvent, type TraceEventType, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, isTraceEvent, isTraceEventType, listTargetNames, loadEvalCases, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.d.ts

@@ -4,32 +4,6 @@ import * as ai from 'ai';
  * Trace event types for capturing agent execution traces.
  * Provides a normalized, provider-agnostic model for tool-call trajectories.
  */
-/**
- * Supported trace event types.
- */
-type TraceEventType = 'model_step' | 'tool_call' | 'tool_result' | 'message' | 'error';
-/**
- * Normalized trace event representing a single step in agent execution.
- * Provider-agnostic format for tool-call trajectory evaluation.
- */
-interface TraceEvent {
-    /** Event type */
-    readonly type: TraceEventType;
-    /** ISO 8601 timestamp */
-    readonly timestamp: string;
-    /** Stable identifier for pairing tool_call/tool_result */
-    readonly id?: string;
-    /** Tool name (for tool_call/tool_result) */
-    readonly name?: string;
-    /** Tool input - any JSON value */
-    readonly input?: unknown;
-    /** Tool output - any JSON value */
-    readonly output?: unknown;
-    /** Message content (for message/model_step) */
-    readonly text?: string;
-    /** Provider-specific metadata */
-    readonly metadata?: Record<string, unknown>;
-}
 /**
  * Compact summary of a trace for lightweight persistence.
  * Included in results by default to avoid payload bloat.
@@ -66,18 +40,19 @@ interface ToolTrajectoryExpectedItem {
     readonly tool: string;
 }
 /**
- * Type guard for TraceEventType values.
+ * Simplified input type for computeTraceSummary.
+ * Matches OutputMessage structure without requiring full provider/types import.
  */
-declare function isTraceEventType(value: unknown): value is TraceEventType;
-/**
- * Type guard for TraceEvent objects.
- */
-declare function isTraceEvent(value: unknown): value is TraceEvent;
+interface OutputMessageLike {
+    readonly toolCalls?: readonly {
+        readonly tool: string;
+    }[];
+}
 /**
- * Compute a lightweight summary from a full trace.
+ * Compute a lightweight summary from output messages.
  * Used for default result persistence without payload bloat.
  */
-declare function computeTraceSummary(trace: readonly TraceEvent[]): TraceSummary;
+declare function computeTraceSummary(messages: readonly OutputMessageLike[]): TraceSummary;
 
 /**
  * JSON primitive values appearing in AgentV payloads.
@@ -105,7 +80,7 @@ type TestMessageRole = (typeof TEST_MESSAGE_ROLE_VALUES)[number];
 /**
  * Text or structured payload attached to a message.
  */
-type TestMessageContent = string | readonly JsonObject[];
+type TestMessageContent = string | JsonObject | readonly JsonObject[];
 /**
  * System-authored instruction message.
  */
@@ -246,8 +221,6 @@ interface EvaluationResult {
     readonly error?: string;
     /** Lightweight summary of the execution trace (always included when available) */
     readonly trace_summary?: TraceSummary;
-    /** Full trace events (only included when --include-trace flag is set) */
-    readonly trace?: readonly TraceEvent[];
 }
 type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
 interface EvaluatorResult {
@@ -290,15 +263,45 @@ interface ProviderRequest {
     readonly metadata?: JsonObject;
     readonly signal?: AbortSignal;
 }
+/**
+ * A tool call within an output message.
+ * Represents a single tool invocation with its input and optional output.
+ */
+interface ToolCall {
+    /** Tool name */
+    readonly tool: string;
+    /** Tool input arguments */
+    readonly input?: unknown;
+    /** Tool output result */
+    readonly output?: unknown;
+    /** Stable identifier for pairing tool calls */
+    readonly id?: string;
+    /** ISO 8601 timestamp */
+    readonly timestamp?: string;
+}
+/**
+ * An output message from agent execution.
+ * Represents a single message in the conversation with optional tool calls.
+ */
+interface OutputMessage {
+    /** Message role (e.g., 'assistant', 'user', 'tool') */
+    readonly role: string;
+    /** Optional name for the message sender */
+    readonly name?: string;
+    /** Message content */
+    readonly content?: unknown;
+    /** Tool calls made in this message */
+    readonly toolCalls?: readonly ToolCall[];
+    /** ISO 8601 timestamp */
+    readonly timestamp?: string;
+    /** Provider-specific metadata */
+    readonly metadata?: Record<string, unknown>;
+}
 interface ProviderResponse {
-    readonly text: string;
-    readonly reasoning?: string;
     readonly raw?: unknown;
     readonly usage?: JsonObject;
-    /** Normalized trace events from agent execution */
-    readonly trace?: readonly TraceEvent[];
-    /** Reference to external trace file (alternative to inline trace) */
-    readonly traceRef?: string;
+    /** Output messages from agent execution (primary source for tool trajectory) */
+    readonly outputMessages?: readonly OutputMessage[];
 }
 interface Provider {
     readonly id: string;
@@ -532,8 +535,6 @@ interface MockResolvedConfig {
     readonly delayMs?: number;
     readonly delayMinMs?: number;
     readonly delayMaxMs?: number;
-    /** Mock trace events for testing tool_trajectory evaluator */
-    readonly trace?: readonly TraceEvent[];
 }
 interface VSCodeResolvedConfig {
     readonly command: string;
@@ -559,6 +560,7 @@ interface CliResolvedConfig {
     readonly timeoutMs?: number;
     readonly healthcheck?: CliHealthcheck;
     readonly verbose?: boolean;
+    readonly keepTempFiles?: boolean;
 }
 type ResolvedTarget = {
     readonly kind: 'azure';
@@ -662,12 +664,10 @@ interface EvaluationContext {
     readonly judgeProvider?: Provider;
     readonly evaluatorTemplateOverride?: string;
     readonly evaluator?: EvaluatorConfig;
-    /** Normalized trace events from provider execution (if available) */
-    readonly candidateTrace?: readonly TraceEvent[];
-    /** File path to trace data (alternative to inline candidateTrace) */
-    readonly candidateTraceRef?: string;
+    /** Output messages from agent execution (primary source for tool trajectory) */
+    readonly outputMessages?: readonly OutputMessage[];
     /** Lightweight summary of trace events (if available) */
-    readonly candidateTraceSummary?: TraceSummary;
+    readonly traceSummary?: TraceSummary;
 }
 interface EvaluationScore {
     readonly score: number;
@@ -737,6 +737,14 @@ declare class ToolTrajectoryEvaluator implements Evaluator {
     private readonly config;
     constructor(options: ToolTrajectoryEvaluatorOptions);
     evaluate(context: EvaluationContext): EvaluationScore;
+    /**
+     * Extract tool calls from output messages.
+     */
+    private extractToolCallsFromMessages;
+    /**
+     * Build a summary from extracted tool calls.
+     */
+    private buildSummary;
     private evaluateAnyOrder;
     private evaluateInOrder;
     private evaluateExact;
@@ -831,4 +839,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceEvent, type TraceEventType, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, isTraceEvent, isTraceEventType, listTargetNames, loadEvalCases, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };