@agentv/core 1.4.0 → 1.5.0

package/dist/index.d.cts

@@ -1,9 +1,21 @@
+import { z } from 'zod';
 import * as ai from 'ai';
 
 /**
  * Trace event types for capturing agent execution traces.
  * Provides a normalized, provider-agnostic model for tool-call trajectories.
  */
+/**
+ * Token usage metrics from provider execution.
+ */
+interface TokenUsage {
+    /** Input/prompt tokens consumed */
+    readonly input: number;
+    /** Output/completion tokens generated */
+    readonly output: number;
+    /** Cached tokens (optional, provider-specific) */
+    readonly cached?: number;
+}
 /**
  * Compact summary of a trace for lightweight persistence.
  * Included in results by default to avoid payload bloat.
@@ -17,6 +29,14 @@ interface TraceSummary {
     readonly toolCallsByName: Readonly<Record<string, number>>;
     /** Number of error events */
     readonly errorCount: number;
+    /** Token usage metrics (optional, from provider) */
+    readonly tokenUsage?: TokenUsage;
+    /** Total cost in USD (optional, from provider) */
+    readonly costUsd?: number;
+    /** Total execution duration in milliseconds (optional) */
+    readonly durationMs?: number;
+    /** Per-tool duration arrays in milliseconds (optional) */
+    readonly toolDurations?: Readonly<Record<string, readonly number[]>>;
 }
 /**
  * Configuration for tool_trajectory evaluator.
@@ -38,6 +58,8 @@ interface ToolTrajectoryEvaluatorConfig {
  */
 interface ToolTrajectoryExpectedItem {
     readonly tool: string;
+    /** Optional argument matching: 'any' skips validation, object performs partial deep equality */
+    readonly args?: 'any' | Record<string, unknown>;
 }
 /**
  * Simplified input type for computeTraceSummary.
@@ -53,6 +75,53 @@ interface OutputMessageLike {
  * Used for default result persistence without payload bloat.
  */
 declare function computeTraceSummary(messages: readonly OutputMessageLike[]): TraceSummary;
+/**
+ * Default tool names considered as exploration/read-only operations.
+ * Can be overridden per-evaluation via config.
+ */
+declare const DEFAULT_EXPLORATION_TOOLS: readonly ["read", "grep", "glob", "search", "list", "Read", "Grep", "Glob", "WebSearch", "WebFetch"];
+/**
+ * Ratio of exploration tool calls to total tool calls.
+ * Returns undefined if there are no tool calls.
+ *
+ * @param summary - Trace summary with tool call counts
+ * @param explorationTools - Tool names considered exploration (defaults to DEFAULT_EXPLORATION_TOOLS)
+ * @returns Ratio between 0 and 1, or undefined if no tool calls
+ */
+declare function explorationRatio(summary: TraceSummary, explorationTools?: readonly string[]): number | undefined;
+/**
+ * Average tokens consumed per tool call.
+ * Returns undefined if tokenUsage is not available or no tool calls.
+ *
+ * @param summary - Trace summary with optional token usage
+ * @returns Average tokens per tool call, or undefined
+ */
+declare function tokensPerTool(summary: TraceSummary): number | undefined;
+/**
+ * Average tool duration across all tool calls.
+ * Returns undefined if toolDurations is not available or empty.
+ *
+ * @param summary - Trace summary with optional tool durations
+ * @returns Average duration in milliseconds, or undefined
+ */
+declare function avgToolDurationMs(summary: TraceSummary): number | undefined;
+/**
+ * Execution metrics from provider response.
+ */
+interface ExecutionMetrics {
+    readonly tokenUsage?: TokenUsage;
+    readonly costUsd?: number;
+    readonly durationMs?: number;
+}
+/**
+ * Merge execution metrics from provider response into a trace summary.
+ * Returns a new TraceSummary with metrics fields populated.
+ *
+ * @param summary - Base trace summary from computeTraceSummary
+ * @param metrics - Optional execution metrics from provider
+ * @returns TraceSummary with merged metrics
+ */
+declare function mergeExecutionMetrics(summary: TraceSummary, metrics?: ExecutionMetrics): TraceSummary;
 
 /**
  * JSON primitive values appearing in AgentV payloads.
@@ -204,23 +273,23 @@ interface EvalCase {
  */
 interface EvaluationResult {
     readonly timestamp: string;
-    readonly eval_id: string;
+    readonly evalId: string;
     readonly dataset?: string;
-    readonly conversation_id?: string;
+    readonly conversationId?: string;
     readonly score: number;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
-    readonly candidate_answer: string;
+    readonly candidateAnswer: string;
     readonly target: string;
     readonly reasoning?: string;
-    readonly raw_aspects?: readonly string[];
-    readonly agent_provider_request?: JsonObject;
-    readonly lm_provider_request?: JsonObject;
-    readonly evaluator_provider_request?: JsonObject;
-    readonly evaluator_results?: readonly EvaluatorResult[];
+    readonly rawAspects?: readonly string[];
+    readonly agentProviderRequest?: JsonObject;
+    readonly lmProviderRequest?: JsonObject;
+    readonly evaluatorProviderRequest?: JsonObject;
+    readonly evaluatorResults?: readonly EvaluatorResult[];
     readonly error?: string;
     /** Lightweight summary of the execution trace (always included when available) */
-    readonly trace_summary?: TraceSummary;
+    readonly traceSummary?: TraceSummary;
 }
 type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
 interface EvaluatorResult {
@@ -232,9 +301,9 @@ interface EvaluatorResult {
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly reasoning?: string;
-    readonly raw_request?: JsonObject;
-    readonly evaluator_provider_request?: JsonObject;
-    readonly evaluator_results?: readonly EvaluatorResult[];
+    readonly rawRequest?: JsonObject;
+    readonly evaluatorProviderRequest?: JsonObject;
+    readonly evaluatorResults?: readonly EvaluatorResult[];
 }
 /**
  * Convenience accessor matching the Python hit_count property.
@@ -248,7 +317,7 @@ interface ChatMessage {
     readonly name?: string;
 }
 type ChatPrompt = readonly ChatMessage[];
-type ProviderKind = 'azure' | 'anthropic' | 'gemini' | 'codex' | 'cli' | 'mock' | 'vscode' | 'vscode-insiders';
+type ProviderKind = 'azure' | 'anthropic' | 'gemini' | 'codex' | 'pi-coding-agent' | 'cli' | 'mock' | 'vscode' | 'vscode-insiders';
 interface ProviderRequest {
     readonly question: string;
     readonly systemPrompt?: string;
@@ -297,11 +366,28 @@ interface OutputMessage {
     /** Provider-specific metadata */
     readonly metadata?: Record<string, unknown>;
 }
+/**
+ * Token usage metrics reported by provider.
+ */
+interface ProviderTokenUsage {
+    /** Input/prompt tokens consumed */
+    readonly input: number;
+    /** Output/completion tokens generated */
+    readonly output: number;
+    /** Cached tokens (optional, provider-specific) */
+    readonly cached?: number;
+}
 interface ProviderResponse {
     readonly raw?: unknown;
     readonly usage?: JsonObject;
     /** Output messages from agent execution (primary source for tool trajectory) */
     readonly outputMessages?: readonly OutputMessage[];
+    /** Token usage metrics (optional) */
+    readonly tokenUsage?: ProviderTokenUsage;
+    /** Total cost in USD (optional) */
+    readonly costUsd?: number;
+    /** Execution duration in milliseconds (optional) */
+    readonly durationMs?: number;
 }
 interface Provider {
     readonly id: string;
@@ -482,6 +568,101 @@ declare function resolveFileReference(rawValue: string, searchRoots: readonly st
     readonly attempted: readonly string[];
 }>;
 
+/**
+ * Strict normalized schema for CLI target configuration.
+ * This is the final validated shape after environment variable resolution
+ * and snake_case to camelCase normalization.
+ *
+ * Uses .strict() to reject unknown properties, ensuring configuration
+ * errors are caught early rather than silently ignored.
+ *
+ * @example
+ * ```typescript
+ * const config: CliNormalizedConfig = {
+ *   commandTemplate: 'agent run {PROMPT}',
+ *   timeoutMs: 120000,
+ *   verbose: true,
+ * };
+ * CliTargetConfigSchema.parse(config); // Validates the normalized config
+ * ```
+ */
+declare const CliTargetConfigSchema: z.ZodObject<{
+    commandTemplate: z.ZodString;
+    filesFormat: z.ZodOptional<z.ZodString>;
+    cwd: z.ZodOptional<z.ZodString>;
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
+    healthcheck: z.ZodOptional<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+        type: z.ZodLiteral<"http">;
+        url: z.ZodString;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, "strict", z.ZodTypeAny, {
+        type: "http";
+        url: string;
+        timeoutMs?: number | undefined;
+    }, {
+        type: "http";
+        url: string;
+        timeoutMs?: number | undefined;
+    }>, z.ZodObject<{
+        type: z.ZodLiteral<"command">;
+        commandTemplate: z.ZodString;
+        cwd: z.ZodOptional<z.ZodString>;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, "strict", z.ZodTypeAny, {
+        type: "command";
+        commandTemplate: string;
+        cwd?: string | undefined;
+        timeoutMs?: number | undefined;
+    }, {
+        type: "command";
+        commandTemplate: string;
+        cwd?: string | undefined;
+        timeoutMs?: number | undefined;
+    }>]>>;
+    verbose: z.ZodOptional<z.ZodBoolean>;
+    keepTempFiles: z.ZodOptional<z.ZodBoolean>;
+}, "strict", z.ZodTypeAny, {
+    commandTemplate: string;
+    cwd?: string | undefined;
+    verbose?: boolean | undefined;
+    filesFormat?: string | undefined;
+    healthcheck?: {
+        type: "http";
+        url: string;
+        timeoutMs?: number | undefined;
+    } | {
+        type: "command";
+        commandTemplate: string;
+        cwd?: string | undefined;
+        timeoutMs?: number | undefined;
+    } | undefined;
+    keepTempFiles?: boolean | undefined;
+    timeoutMs?: number | undefined;
+}, {
+    commandTemplate: string;
+    cwd?: string | undefined;
+    verbose?: boolean | undefined;
+    filesFormat?: string | undefined;
+    healthcheck?: {
+        type: "http";
+        url: string;
+        timeoutMs?: number | undefined;
+    } | {
+        type: "command";
+        commandTemplate: string;
+        cwd?: string | undefined;
+        timeoutMs?: number | undefined;
+    } | undefined;
+    keepTempFiles?: boolean | undefined;
+    timeoutMs?: number | undefined;
+}>;
+type CliNormalizedConfig = z.infer<typeof CliTargetConfigSchema>;
+/**
+ * Resolved CLI configuration type derived from CliTargetConfigSchema.
+ * This is the final validated shape used by the CLI provider at runtime.
+ * Using Readonly to ensure immutability for runtime safety.
+ */
+type CliResolvedConfig = Readonly<CliNormalizedConfig>;
 interface RetryConfig {
     readonly maxRetries?: number;
     readonly initialDelayMs?: number;
@@ -529,6 +710,21 @@ interface CodexResolvedConfig {
     readonly timeoutMs?: number;
     readonly logDir?: string;
     readonly logFormat?: 'summary' | 'json';
+    readonly systemPrompt?: string;
+}
+interface PiCodingAgentResolvedConfig {
+    readonly executable: string;
+    readonly provider?: string;
+    readonly model?: string;
+    readonly apiKey?: string;
+    readonly tools?: string;
+    readonly thinking?: string;
+    readonly args?: readonly string[];
+    readonly cwd?: string;
+    readonly timeoutMs?: number;
+    readonly logDir?: string;
+    readonly logFormat?: 'summary' | 'json';
+    readonly systemPrompt?: string;
 }
 interface MockResolvedConfig {
     readonly response?: string;
@@ -543,25 +739,6 @@ interface VSCodeResolvedConfig {
     readonly subagentRoot?: string;
     readonly workspaceTemplate?: string;
 }
-type CliHealthcheck = {
-    readonly type: 'http';
-    readonly url: string;
-    readonly timeoutMs?: number;
-} | {
-    readonly type: 'command';
-    readonly commandTemplate: string;
-    readonly timeoutMs?: number;
-    readonly cwd?: string;
-};
-interface CliResolvedConfig {
-    readonly commandTemplate: string;
-    readonly filesFormat?: string;
-    readonly cwd?: string;
-    readonly timeoutMs?: number;
-    readonly healthcheck?: CliHealthcheck;
-    readonly verbose?: boolean;
-    readonly keepTempFiles?: boolean;
-}
 type ResolvedTarget = {
     readonly kind: 'azure';
     readonly name: string;
@@ -590,6 +767,13 @@ type ResolvedTarget = {
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: CodexResolvedConfig;
+} | {
+    readonly kind: 'pi-coding-agent';
+    readonly name: string;
+    readonly judgeTarget?: string;
+    readonly workers?: number;
+    readonly providerBatching?: boolean;
+    readonly config: PiCodingAgentResolvedConfig;
 } | {
     readonly kind: 'mock';
     readonly name: string;
@@ -645,6 +829,16 @@ type CodexLogListener = (entry: CodexLogEntry) => void;
 declare function consumeCodexLogEntries(): CodexLogEntry[];
 declare function subscribeToCodexLogEntries(listener: CodexLogListener): () => void;
 
+type PiLogEntry = {
+    readonly filePath: string;
+    readonly evalCaseId?: string;
+    readonly targetName: string;
+    readonly attempt?: number;
+};
+type PiLogListener = (entry: PiLogEntry) => void;
+declare function consumePiLogEntries(): PiLogEntry[];
+declare function subscribeToPiLogEntries(listener: PiLogListener): () => void;
+
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
@@ -839,4 +1033,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, DEFAULT_EXPLORATION_TOOLS, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type ExecutionMetrics, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type PiCodingAgentResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ProviderTokenUsage, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, avgToolDurationMs, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, consumePiLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, explorationRatio, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, mergeExecutionMetrics, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries, subscribeToPiLogEntries, tokensPerTool };