@agentv/core 1.3.1 → 1.5.0

package/dist/index.d.cts

@@ -1,3 +1,4 @@
+import { z } from 'zod';
 import * as ai from 'ai';
 
 /**
@@ -5,30 +6,15 @@ import * as ai from 'ai';
  * Provides a normalized, provider-agnostic model for tool-call trajectories.
  */
 /**
- * Supported trace event types.
+ * Token usage metrics from provider execution.
  */
-type TraceEventType = 'model_step' | 'tool_call' | 'tool_result' | 'message' | 'error';
-/**
- * Normalized trace event representing a single step in agent execution.
- * Provider-agnostic format for tool-call trajectory evaluation.
- */
-interface TraceEvent {
-    /** Event type */
-    readonly type: TraceEventType;
-    /** ISO 8601 timestamp */
-    readonly timestamp: string;
-    /** Stable identifier for pairing tool_call/tool_result */
-    readonly id?: string;
-    /** Tool name (for tool_call/tool_result) */
-    readonly name?: string;
-    /** Tool input - any JSON value */
-    readonly input?: unknown;
-    /** Tool output - any JSON value */
-    readonly output?: unknown;
-    /** Message content (for message/model_step) */
-    readonly text?: string;
-    /** Provider-specific metadata */
-    readonly metadata?: Record<string, unknown>;
+interface TokenUsage {
+    /** Input/prompt tokens consumed */
+    readonly input: number;
+    /** Output/completion tokens generated */
+    readonly output: number;
+    /** Cached tokens (optional, provider-specific) */
+    readonly cached?: number;
 }
 /**
  * Compact summary of a trace for lightweight persistence.
@@ -43,6 +29,14 @@ interface TraceSummary {
     readonly toolCallsByName: Readonly<Record<string, number>>;
     /** Number of error events */
     readonly errorCount: number;
+    /** Token usage metrics (optional, from provider) */
+    readonly tokenUsage?: TokenUsage;
+    /** Total cost in USD (optional, from provider) */
+    readonly costUsd?: number;
+    /** Total execution duration in milliseconds (optional) */
+    readonly durationMs?: number;
+    /** Per-tool duration arrays in milliseconds (optional) */
+    readonly toolDurations?: Readonly<Record<string, readonly number[]>>;
 }
 /**
  * Configuration for tool_trajectory evaluator.
@@ -64,20 +58,70 @@ interface ToolTrajectoryEvaluatorConfig {
  */
 interface ToolTrajectoryExpectedItem {
     readonly tool: string;
+    /** Optional argument matching: 'any' skips validation, object performs partial deep equality */
+    readonly args?: 'any' | Record<string, unknown>;
 }
 /**
- * Type guard for TraceEventType values.
+ * Simplified input type for computeTraceSummary.
+ * Matches OutputMessage structure without requiring full provider/types import.
  */
-declare function isTraceEventType(value: unknown): value is TraceEventType;
+interface OutputMessageLike {
+    readonly toolCalls?: readonly {
+        readonly tool: string;
+    }[];
+}
 /**
- * Type guard for TraceEvent objects.
+ * Compute a lightweight summary from output messages.
+ * Used for default result persistence without payload bloat.
  */
-declare function isTraceEvent(value: unknown): value is TraceEvent;
+declare function computeTraceSummary(messages: readonly OutputMessageLike[]): TraceSummary;
 /**
- * Compute a lightweight summary from a full trace.
- * Used for default result persistence without payload bloat.
+ * Default tool names considered as exploration/read-only operations.
+ * Can be overridden per-evaluation via config.
+ */
+declare const DEFAULT_EXPLORATION_TOOLS: readonly ["read", "grep", "glob", "search", "list", "Read", "Grep", "Glob", "WebSearch", "WebFetch"];
+/**
+ * Ratio of exploration tool calls to total tool calls.
+ * Returns undefined if there are no tool calls.
+ *
+ * @param summary - Trace summary with tool call counts
+ * @param explorationTools - Tool names considered exploration (defaults to DEFAULT_EXPLORATION_TOOLS)
+ * @returns Ratio between 0 and 1, or undefined if no tool calls
+ */
+declare function explorationRatio(summary: TraceSummary, explorationTools?: readonly string[]): number | undefined;
+/**
+ * Average tokens consumed per tool call.
+ * Returns undefined if tokenUsage is not available or no tool calls.
+ *
+ * @param summary - Trace summary with optional token usage
+ * @returns Average tokens per tool call, or undefined
+ */
+declare function tokensPerTool(summary: TraceSummary): number | undefined;
+/**
+ * Average tool duration across all tool calls.
+ * Returns undefined if toolDurations is not available or empty.
+ *
+ * @param summary - Trace summary with optional tool durations
+ * @returns Average duration in milliseconds, or undefined
+ */
+declare function avgToolDurationMs(summary: TraceSummary): number | undefined;
+/**
+ * Execution metrics from provider response.
+ */
+interface ExecutionMetrics {
+    readonly tokenUsage?: TokenUsage;
+    readonly costUsd?: number;
+    readonly durationMs?: number;
+}
+/**
+ * Merge execution metrics from provider response into a trace summary.
+ * Returns a new TraceSummary with metrics fields populated.
+ *
+ * @param summary - Base trace summary from computeTraceSummary
+ * @param metrics - Optional execution metrics from provider
+ * @returns TraceSummary with merged metrics
  */
-declare function computeTraceSummary(trace: readonly TraceEvent[]): TraceSummary;
+declare function mergeExecutionMetrics(summary: TraceSummary, metrics?: ExecutionMetrics): TraceSummary;
 
 /**
  * JSON primitive values appearing in AgentV payloads.
@@ -229,25 +273,23 @@ interface EvalCase {
  */
 interface EvaluationResult {
     readonly timestamp: string;
-    readonly eval_id: string;
+    readonly evalId: string;
     readonly dataset?: string;
-    readonly conversation_id?: string;
+    readonly conversationId?: string;
     readonly score: number;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
-    readonly candidate_answer: string;
+    readonly candidateAnswer: string;
     readonly target: string;
     readonly reasoning?: string;
-    readonly raw_aspects?: readonly string[];
-    readonly agent_provider_request?: JsonObject;
-    readonly lm_provider_request?: JsonObject;
-    readonly evaluator_provider_request?: JsonObject;
-    readonly evaluator_results?: readonly EvaluatorResult[];
+    readonly rawAspects?: readonly string[];
+    readonly agentProviderRequest?: JsonObject;
+    readonly lmProviderRequest?: JsonObject;
+    readonly evaluatorProviderRequest?: JsonObject;
+    readonly evaluatorResults?: readonly EvaluatorResult[];
     readonly error?: string;
     /** Lightweight summary of the execution trace (always included when available) */
-    readonly trace_summary?: TraceSummary;
-    /** Full trace events (only included when --include-trace flag is set) */
-    readonly trace?: readonly TraceEvent[];
+    readonly traceSummary?: TraceSummary;
 }
 type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
 interface EvaluatorResult {
@@ -259,9 +301,9 @@ interface EvaluatorResult {
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly reasoning?: string;
-    readonly raw_request?: JsonObject;
-    readonly evaluator_provider_request?: JsonObject;
-    readonly evaluator_results?: readonly EvaluatorResult[];
+    readonly rawRequest?: JsonObject;
+    readonly evaluatorProviderRequest?: JsonObject;
+    readonly evaluatorResults?: readonly EvaluatorResult[];
 }
 /**
  * Convenience accessor matching the Python hit_count property.
@@ -275,7 +317,7 @@ interface ChatMessage {
     readonly name?: string;
 }
 type ChatPrompt = readonly ChatMessage[];
-type ProviderKind = 'azure' | 'anthropic' | 'gemini' | 'codex' | 'cli' | 'mock' | 'vscode' | 'vscode-insiders';
+type ProviderKind = 'azure' | 'anthropic' | 'gemini' | 'codex' | 'pi-coding-agent' | 'cli' | 'mock' | 'vscode' | 'vscode-insiders';
 interface ProviderRequest {
     readonly question: string;
     readonly systemPrompt?: string;
@@ -290,15 +332,62 @@ interface ProviderRequest {
     readonly metadata?: JsonObject;
     readonly signal?: AbortSignal;
 }
+/**
+ * A tool call within an output message.
+ * Represents a single tool invocation with its input and optional output.
+ */
+interface ToolCall {
+    /** Tool name */
+    readonly tool: string;
+    /** Tool input arguments */
+    readonly input?: unknown;
+    /** Tool output result */
+    readonly output?: unknown;
+    /** Stable identifier for pairing tool calls */
+    readonly id?: string;
+    /** ISO 8601 timestamp */
+    readonly timestamp?: string;
+}
+/**
+ * An output message from agent execution.
+ * Represents a single message in the conversation with optional tool calls.
+ */
+interface OutputMessage {
+    /** Message role (e.g., 'assistant', 'user', 'tool') */
+    readonly role: string;
+    /** Optional name for the message sender */
+    readonly name?: string;
+    /** Message content */
+    readonly content?: unknown;
+    /** Tool calls made in this message */
+    readonly toolCalls?: readonly ToolCall[];
+    /** ISO 8601 timestamp */
+    readonly timestamp?: string;
+    /** Provider-specific metadata */
+    readonly metadata?: Record<string, unknown>;
+}
+/**
+ * Token usage metrics reported by provider.
+ */
+interface ProviderTokenUsage {
+    /** Input/prompt tokens consumed */
+    readonly input: number;
+    /** Output/completion tokens generated */
+    readonly output: number;
+    /** Cached tokens (optional, provider-specific) */
+    readonly cached?: number;
+}
 interface ProviderResponse {
-    readonly text: string;
-    readonly reasoning?: string;
     readonly raw?: unknown;
     readonly usage?: JsonObject;
-    /** Normalized trace events from agent execution */
-    readonly trace?: readonly TraceEvent[];
-    /** Reference to external trace file (alternative to inline trace) */
-    readonly traceRef?: string;
+    /** Output messages from agent execution (primary source for tool trajectory) */
+    readonly outputMessages?: readonly OutputMessage[];
+    /** Token usage metrics (optional) */
+    readonly tokenUsage?: ProviderTokenUsage;
+    /** Total cost in USD (optional) */
+    readonly costUsd?: number;
+    /** Execution duration in milliseconds (optional) */
+    readonly durationMs?: number;
 }
 interface Provider {
     readonly id: string;
@@ -479,6 +568,101 @@ declare function resolveFileReference(rawValue: string, searchRoots: readonly st
     readonly attempted: readonly string[];
 }>;
 
+/**
+ * Strict normalized schema for CLI target configuration.
+ * This is the final validated shape after environment variable resolution
+ * and snake_case to camelCase normalization.
+ *
+ * Uses .strict() to reject unknown properties, ensuring configuration
+ * errors are caught early rather than silently ignored.
+ *
+ * @example
+ * ```typescript
+ * const config: CliNormalizedConfig = {
+ *   commandTemplate: 'agent run {PROMPT}',
+ *   timeoutMs: 120000,
+ *   verbose: true,
+ * };
+ * CliTargetConfigSchema.parse(config); // Validates the normalized config
+ * ```
+ */
+declare const CliTargetConfigSchema: z.ZodObject<{
+    commandTemplate: z.ZodString;
+    filesFormat: z.ZodOptional<z.ZodString>;
+    cwd: z.ZodOptional<z.ZodString>;
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
+    healthcheck: z.ZodOptional<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+        type: z.ZodLiteral<"http">;
+        url: z.ZodString;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, "strict", z.ZodTypeAny, {
+        type: "http";
+        url: string;
+        timeoutMs?: number | undefined;
+    }, {
+        type: "http";
+        url: string;
+        timeoutMs?: number | undefined;
+    }>, z.ZodObject<{
+        type: z.ZodLiteral<"command">;
+        commandTemplate: z.ZodString;
+        cwd: z.ZodOptional<z.ZodString>;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, "strict", z.ZodTypeAny, {
+        type: "command";
+        commandTemplate: string;
+        cwd?: string | undefined;
+        timeoutMs?: number | undefined;
+    }, {
+        type: "command";
+        commandTemplate: string;
+        cwd?: string | undefined;
+        timeoutMs?: number | undefined;
+    }>]>>;
+    verbose: z.ZodOptional<z.ZodBoolean>;
+    keepTempFiles: z.ZodOptional<z.ZodBoolean>;
+}, "strict", z.ZodTypeAny, {
+    commandTemplate: string;
+    cwd?: string | undefined;
+    verbose?: boolean | undefined;
+    filesFormat?: string | undefined;
+    healthcheck?: {
+        type: "http";
+        url: string;
+        timeoutMs?: number | undefined;
+    } | {
+        type: "command";
+        commandTemplate: string;
+        cwd?: string | undefined;
+        timeoutMs?: number | undefined;
+    } | undefined;
+    keepTempFiles?: boolean | undefined;
+    timeoutMs?: number | undefined;
+}, {
+    commandTemplate: string;
+    cwd?: string | undefined;
+    verbose?: boolean | undefined;
+    filesFormat?: string | undefined;
+    healthcheck?: {
+        type: "http";
+        url: string;
+        timeoutMs?: number | undefined;
+    } | {
+        type: "command";
+        commandTemplate: string;
+        cwd?: string | undefined;
+        timeoutMs?: number | undefined;
+    } | undefined;
+    keepTempFiles?: boolean | undefined;
+    timeoutMs?: number | undefined;
+}>;
+type CliNormalizedConfig = z.infer<typeof CliTargetConfigSchema>;
+/**
+ * Resolved CLI configuration type derived from CliTargetConfigSchema.
+ * This is the final validated shape used by the CLI provider at runtime.
+ * Using Readonly to ensure immutability for runtime safety.
+ */
+type CliResolvedConfig = Readonly<CliNormalizedConfig>;
 interface RetryConfig {
     readonly maxRetries?: number;
     readonly initialDelayMs?: number;
@@ -526,14 +710,27 @@ interface CodexResolvedConfig {
     readonly timeoutMs?: number;
     readonly logDir?: string;
     readonly logFormat?: 'summary' | 'json';
+    readonly systemPrompt?: string;
+}
+interface PiCodingAgentResolvedConfig {
+    readonly executable: string;
+    readonly provider?: string;
+    readonly model?: string;
+    readonly apiKey?: string;
+    readonly tools?: string;
+    readonly thinking?: string;
+    readonly args?: readonly string[];
+    readonly cwd?: string;
+    readonly timeoutMs?: number;
+    readonly logDir?: string;
+    readonly logFormat?: 'summary' | 'json';
+    readonly systemPrompt?: string;
 }
 interface MockResolvedConfig {
     readonly response?: string;
     readonly delayMs?: number;
     readonly delayMinMs?: number;
     readonly delayMaxMs?: number;
-    /** Mock trace events for testing tool_trajectory evaluator */
-    readonly trace?: readonly TraceEvent[];
 }
 interface VSCodeResolvedConfig {
     readonly command: string;
@@ -542,24 +739,6 @@ interface VSCodeResolvedConfig {
     readonly subagentRoot?: string;
     readonly workspaceTemplate?: string;
 }
-type CliHealthcheck = {
-    readonly type: 'http';
-    readonly url: string;
-    readonly timeoutMs?: number;
-} | {
-    readonly type: 'command';
-    readonly commandTemplate: string;
-    readonly timeoutMs?: number;
-    readonly cwd?: string;
-};
-interface CliResolvedConfig {
-    readonly commandTemplate: string;
-    readonly filesFormat?: string;
-    readonly cwd?: string;
-    readonly timeoutMs?: number;
-    readonly healthcheck?: CliHealthcheck;
-    readonly verbose?: boolean;
-}
 type ResolvedTarget = {
     readonly kind: 'azure';
     readonly name: string;
@@ -588,6 +767,13 @@ type ResolvedTarget = {
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: CodexResolvedConfig;
+} | {
+    readonly kind: 'pi-coding-agent';
+    readonly name: string;
+    readonly judgeTarget?: string;
+    readonly workers?: number;
+    readonly providerBatching?: boolean;
+    readonly config: PiCodingAgentResolvedConfig;
 } | {
     readonly kind: 'mock';
     readonly name: string;
@@ -643,6 +829,16 @@ type CodexLogListener = (entry: CodexLogEntry) => void;
 declare function consumeCodexLogEntries(): CodexLogEntry[];
 declare function subscribeToCodexLogEntries(listener: CodexLogListener): () => void;
 
+type PiLogEntry = {
+    readonly filePath: string;
+    readonly evalCaseId?: string;
+    readonly targetName: string;
+    readonly attempt?: number;
+};
+type PiLogListener = (entry: PiLogEntry) => void;
+declare function consumePiLogEntries(): PiLogEntry[];
+declare function subscribeToPiLogEntries(listener: PiLogListener): () => void;
+
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
@@ -662,12 +858,10 @@ interface EvaluationContext {
     readonly judgeProvider?: Provider;
     readonly evaluatorTemplateOverride?: string;
     readonly evaluator?: EvaluatorConfig;
-    /** Normalized trace events from provider execution (if available) */
-    readonly candidateTrace?: readonly TraceEvent[];
-    /** File path to trace data (alternative to inline candidateTrace) */
-    readonly candidateTraceRef?: string;
+    /** Output messages from agent execution (primary source for tool trajectory) */
+    readonly outputMessages?: readonly OutputMessage[];
     /** Lightweight summary of trace events (if available) */
-    readonly candidateTraceSummary?: TraceSummary;
+    readonly traceSummary?: TraceSummary;
 }
 interface EvaluationScore {
     readonly score: number;
@@ -737,6 +931,14 @@ declare class ToolTrajectoryEvaluator implements Evaluator {
     private readonly config;
     constructor(options: ToolTrajectoryEvaluatorOptions);
     evaluate(context: EvaluationContext): EvaluationScore;
+    /**
+     * Extract tool calls from output messages.
+     */
+    private extractToolCallsFromMessages;
+    /**
+     * Build a summary from extracted tool calls.
+     */
+    private buildSummary;
     private evaluateAnyOrder;
     private evaluateInOrder;
     private evaluateExact;
@@ -831,4 +1033,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceEvent, type TraceEventType, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, isTraceEvent, isTraceEventType, listTargetNames, loadEvalCases, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, DEFAULT_EXPLORATION_TOOLS, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type ExecutionMetrics, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type PiCodingAgentResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ProviderTokenUsage, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, avgToolDurationMs, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, consumePiLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, explorationRatio, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, mergeExecutionMetrics, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries, subscribeToPiLogEntries, tokensPerTool };