@agentv/core 2.7.1-next.6 → 2.9.0-next.1

package/dist/index.d.cts

@@ -187,8 +187,6 @@ interface TargetDefinition {
     readonly subagentRoot?: string | unknown | undefined;
     readonly workspace_template?: string | unknown | undefined;
     readonly workspaceTemplate?: string | unknown | undefined;
-    readonly command_template?: string | unknown | undefined;
-    readonly commandTemplate?: string | unknown | undefined;
     readonly files_format?: string | unknown | undefined;
     readonly filesFormat?: string | unknown | undefined;
     readonly attachments_format?: string | unknown | undefined;
@@ -241,20 +239,22 @@ interface TraceSummary {
     readonly toolCallsByName: Readonly<Record<string, number>>;
     /** Number of error events */
     readonly errorCount: number;
-    /** Token usage metrics (optional, from provider) */
+    /** Per-tool duration arrays in milliseconds (optional) */
+    readonly toolDurations?: Readonly<Record<string, readonly number[]>>;
+    /** Number of LLM calls (assistant messages) */
+    readonly llmCallCount?: number;
+}
+/**
+ * Combined result of trace computation + execution metrics merge.
+ * Returned by computeTraceSummaryWithMetrics().
+ */
+interface TraceComputeResult {
+    readonly trace: TraceSummary;
     readonly tokenUsage?: TokenUsage;
-    /** Total cost in USD (optional, from provider) */
     readonly costUsd?: number;
-    /** Total execution duration in milliseconds (optional) */
     readonly durationMs?: number;
-    /** Per-tool duration arrays in milliseconds (optional) */
-    readonly toolDurations?: Readonly<Record<string, readonly number[]>>;
-    /** ISO 8601 timestamp when execution started (derived from earliest span) */
     readonly startTime?: string;
-    /** ISO 8601 timestamp when execution ended (derived from latest span) */
     readonly endTime?: string;
-    /** Number of LLM calls (assistant messages) */
-    readonly llmCallCount?: number;
 }
 /**
  * Argument matching mode for tool_trajectory expected items.
@@ -321,7 +321,7 @@ interface MessageLike {
  * - toolDurations: per-tool duration arrays (from durationMs or computed from start/end)
  * - llmCallCount: count of assistant messages
  */
-declare function computeTraceSummary(messages: readonly MessageLike[]): TraceSummary;
+declare function computeTraceSummary(messages: readonly MessageLike[]): TraceComputeResult;
 /**
  * Default tool names considered as exploration/read-only operations.
  * Can be overridden per-evaluation via config.
@@ -343,7 +343,7 @@ declare function explorationRatio(summary: TraceSummary, explorationTools?: read
  * @param summary - Trace summary with optional token usage
  * @returns Average tokens per tool call, or undefined
  */
-declare function tokensPerTool(summary: TraceSummary): number | undefined;
+declare function tokensPerTool(summary: TraceSummary, tokenUsage?: TokenUsage): number | undefined;
 /**
  * Average tool duration across all tool calls.
  * Returns undefined if toolDurations is not available or empty.
@@ -365,15 +365,15 @@ interface ExecutionMetrics {
     readonly endTime?: string;
 }
 /**
- * Merge execution metrics from provider response into a trace summary.
- * Returns a new TraceSummary with metrics fields populated.
+ * Merge execution metrics from provider response into a trace compute result.
+ * Returns a new TraceComputeResult with metrics fields populated.
  * Provider-level timing takes precedence over span-derived timing.
  *
- * @param summary - Base trace summary from computeTraceSummary
+ * @param computed - Base trace compute result from computeTraceSummary
  * @param metrics - Optional execution metrics from provider
- * @returns TraceSummary with merged metrics
+ * @returns TraceComputeResult with merged metrics
  */
-declare function mergeExecutionMetrics(summary: TraceSummary, metrics?: ExecutionMetrics): TraceSummary;
+declare function mergeExecutionMetrics(computed: TraceComputeResult, metrics?: ExecutionMetrics): TraceComputeResult;
 
 /**
  * JSON primitive values appearing in AgentV payloads.
@@ -924,6 +924,16 @@ interface EvaluationResult {
     readonly answer: string;
     readonly target: string;
     readonly reasoning?: string;
+    /** Token usage metrics from provider (optional) */
+    readonly tokenUsage?: TokenUsage;
+    /** Total cost in USD (optional, from provider) */
+    readonly costUsd?: number;
+    /** Total execution duration in milliseconds (optional) */
+    readonly durationMs?: number;
+    /** ISO 8601 timestamp when execution started */
+    readonly startTime?: string;
+    /** ISO 8601 timestamp when execution ended */
+    readonly endTime?: string;
     readonly requests?: {
         readonly agent?: JsonObject;
         readonly lm?: JsonObject;
@@ -955,6 +965,8 @@ interface EvaluationResult {
     readonly aggregation?: TrialAggregation;
     /** Whether the trial loop was terminated early due to cost limit */
     readonly costLimited?: boolean;
+    /** Whether the evaluation was skipped due to suite-level budget exhaustion */
+    readonly budgetExceeded?: boolean;
 }
 type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
 interface EvaluatorResult {
@@ -971,6 +983,8 @@ interface EvaluatorResult {
     readonly scores?: readonly EvaluatorResult[];
     /** Optional structured details from code judges (e.g., TP/TN/FP/FN counts). */
     readonly details?: JsonObject;
+    /** Token usage from LLM calls made by this evaluator (optional). */
+    readonly tokenUsage?: TokenUsage;
 }
 /**
  * Convenience accessor matching the Python hit_count property.
@@ -1116,6 +1130,8 @@ type EvalSuiteResult = {
     readonly cacheConfig?: CacheConfig;
     /** Suite-level metadata (name, description, version, etc.) */
     readonly metadata?: EvalMetadata;
+    /** Suite-level total cost budget in USD */
+    readonly totalBudgetUsd?: number;
 };
 /**
  * Load tests and suite metadata from a single parse.
@@ -1637,6 +1653,16 @@ interface EvaluationContext {
     readonly output?: readonly Message[];
     /** Lightweight summary of trace events (if available) */
     readonly trace?: TraceSummary;
+    /** Token usage from provider execution (promoted from TraceSummary) */
+    readonly tokenUsage?: TokenUsage;
+    /** Total cost in USD (from provider) */
+    readonly costUsd?: number;
+    /** Execution duration in milliseconds */
+    readonly durationMs?: number;
+    /** ISO 8601 timestamp when execution started */
+    readonly startTime?: string;
+    /** ISO 8601 timestamp when execution ended */
+    readonly endTime?: string;
     /** Resolver for target override in code judges */
     readonly targetResolver?: TargetResolver;
     /** List of available target names for code judges */
@@ -1657,6 +1683,8 @@ interface EvaluationScore {
     readonly scores?: readonly ChildEvaluatorResult[];
     /** Optional structured details from code judges (e.g., TP/TN/FP/FN counts, alignments). */
     readonly details?: JsonObject;
+    /** Token usage from LLM calls made by this evaluator (optional). */
+    readonly tokenUsage?: TokenUsage;
 }
 interface ChildEvaluatorResult {
     readonly name: string;
@@ -1671,6 +1699,8 @@ interface ChildEvaluatorResult {
     readonly scores?: readonly ChildEvaluatorResult[];
     /** Optional structured details from code judges (e.g., TP/TN/FP/FN counts, alignments). */
     readonly details?: JsonObject;
+    /** Token usage from LLM calls made by this evaluator (optional). */
+    readonly tokenUsage?: TokenUsage;
 }
 interface Evaluator {
     readonly kind: string;
@@ -1740,7 +1770,7 @@ interface CostEvaluatorOptions {
 }
 /**
  * Evaluator that checks execution cost against a budget.
- * Uses trace.costUsd from the evaluation context.
+ * Uses costUsd from the evaluation context.
  */
 declare class CostEvaluator implements Evaluator {
     readonly kind = "cost";
@@ -1812,7 +1842,7 @@ interface LatencyEvaluatorOptions {
 }
 /**
  * Evaluator that checks execution duration against a threshold.
- * Uses trace.durationMs from the evaluation context.
+ * Uses durationMs from the evaluation context.
  */
 declare class LatencyEvaluator implements Evaluator {
     readonly kind = "latency";
@@ -1987,7 +2017,7 @@ interface TokenUsageEvaluatorOptions {
 }
 /**
  * Evaluator that checks provider-reported token usage against configured limits.
- * Uses trace.tokenUsage from the evaluation context.
+ * Uses tokenUsage from the evaluation context.
  */
 declare class TokenUsageEvaluator implements Evaluator {
     readonly kind = "token_usage";
@@ -2196,6 +2226,8 @@ interface RunEvaluationOptions {
     readonly trials?: TrialsConfig;
     /** Real-time observability callbacks passed to the provider */
     readonly streamCallbacks?: ProviderStreamCallbacks;
+    /** Suite-level total cost budget in USD (stops dispatching when exceeded) */
+    readonly totalBudgetUsd?: number;
 }
 declare function runEvaluation(options: RunEvaluationOptions): Promise<readonly EvaluationResult[]>;
 declare function runEvalCase(options: RunEvalCaseOptions): Promise<EvaluationResult>;
@@ -2886,4 +2918,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { AgentJudgeEvaluator, type AgentJudgeEvaluatorConfig, type AgentJudgeEvaluatorOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVConfig as AgentVTsConfig, type AnthropicResolvedConfig, type ArgsMatchMode, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, type CacheConfig, type ChildEvaluatorResult, type ClaudeResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type ConfidenceIntervalAggregation, type ContainsEvaluatorConfig, type CopilotCliResolvedConfig, type CopilotSdkResolvedConfig, CostEvaluator, type CostEvaluatorConfig, type CostEvaluatorOptions, DEFAULT_EVALUATOR_TEMPLATE, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DeterministicAssertionEvaluator, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsEvaluatorConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTest, type EvalTestInput, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorDispatchContext, type EvaluatorFactory, type EvaluatorFactoryFn, type EvaluatorKind, EvaluatorRegistry, type EvaluatorResult, type ExecutionMetrics, ExecutionMetricsEvaluator, type ExecutionMetricsEvaluatorConfig, type ExecutionMetricsEvaluatorOptions, FieldAccuracyEvaluator, type FieldAccuracyEvaluatorConfig, type FieldAccuracyEvaluatorOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type IsJsonEvaluatorConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyEvaluator, type LatencyEvaluatorConfig, type LatencyEvaluatorOptions, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type LlmJudgePromptAssembly, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, type PassAtKAggregation, type PiAgentSdkResolvedConfig, type PiCodingAgentResolvedConfig, type ProgressEvent, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexEvaluatorConfig, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type RubricItem, type RubricsEvaluatorConfig, type RunEvalCaseOptions, type RunEvaluationOptions, type ScoreRange, type ScriptExecutionContext, SimpleTraceFileExporter, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetAccessConfig, type TargetDefinition, TemplateNotDirectoryError, TemplateNotFoundError, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, TokenUsageEvaluator, type TokenUsageEvaluatorConfig, type TokenUsageEvaluatorOptions, type ToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceSummary, type TrialAggregation, type TrialResult, type TrialStrategy, type TrialsConfig, type UserTestMessage, type VSCodeResolvedConfig, type WorkspaceConfig, WorkspaceCreationError, type WorkspaceScriptConfig, assembleLlmJudgePrompt, avgToolDurationMs, buildDirectoryChain, buildOutputSchema, buildPromptInputs, buildRubricOutputSchema, buildScoreRangeOutputSchema, buildSearchRoots, calculateRubricScore, captureFileChanges, clampScore, cleanupEvalWorkspaces, cleanupWorkspace, computeTraceSummary, consumeClaudeLogEntries, consumeCodexLogEntries, consumeCopilotCliLogEntries, consumeCopilotSdkLogEntries, consumePiLogEntries, createAgentKernel, createBuiltinProviderRegistry, createBuiltinRegistry, createProvider, createTempWorkspace, deepEqual, defineConfig, detectFormat, discoverAssertions, discoverProviders, ensureVSCodeSubagents, evaluate, executeScript, executeWorkspaceScript, explorationRatio, extractCacheConfig, extractJsonBlob, extractTargetFromSuite, extractTargetsFromSuite, extractTargetsFromTestCase, extractTrialsConfig, fileExists, findGitRoot, freeformEvaluationSchema, generateRubrics, getHitCount, getWorkspacePath, initializeBaseline, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isNonEmptyString, isTestMessage, isTestMessageRole, listTargetNames, loadConfig, loadEvalCaseById, loadEvalCases, loadEvalSuite, loadTestById, loadTestSuite, loadTests, loadTsConfig, mergeExecutionMetrics, negateScore, normalizeLineEndings, parseJsonFromText, parseJsonSafe, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, resolveWorkspaceTemplate, rubricEvaluationSchema, runContainsAssertion, runEqualsAssertion, runEvalCase, runEvaluation, runIsJsonAssertion, runRegexAssertion, scoreToVerdict, shouldEnableCache, shouldSkipCacheForTemperature, subscribeToClaudeLogEntries, subscribeToCodexLogEntries, subscribeToCopilotCliLogEntries, subscribeToCopilotSdkLogEntries, subscribeToPiLogEntries, substituteVariables, toCamelCaseDeep, toSnakeCaseDeep, tokensPerTool, trimBaselineResult };
+export { AgentJudgeEvaluator, type AgentJudgeEvaluatorConfig, type AgentJudgeEvaluatorOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVConfig as AgentVTsConfig, type AnthropicResolvedConfig, type ArgsMatchMode, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, type CacheConfig, type ChildEvaluatorResult, type ClaudeResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type ConfidenceIntervalAggregation, type ContainsEvaluatorConfig, type CopilotCliResolvedConfig, type CopilotSdkResolvedConfig, CostEvaluator, type CostEvaluatorConfig, type CostEvaluatorOptions, DEFAULT_EVALUATOR_TEMPLATE, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DeterministicAssertionEvaluator, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsEvaluatorConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTest, type EvalTestInput, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorDispatchContext, type EvaluatorFactory, type EvaluatorFactoryFn, type EvaluatorKind, EvaluatorRegistry, type EvaluatorResult, type ExecutionMetrics, ExecutionMetricsEvaluator, type ExecutionMetricsEvaluatorConfig, type ExecutionMetricsEvaluatorOptions, FieldAccuracyEvaluator, type FieldAccuracyEvaluatorConfig, type FieldAccuracyEvaluatorOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type IsJsonEvaluatorConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyEvaluator, type LatencyEvaluatorConfig, type LatencyEvaluatorOptions, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type LlmJudgePromptAssembly, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, type PassAtKAggregation, type PiAgentSdkResolvedConfig, type PiCodingAgentResolvedConfig, type ProgressEvent, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexEvaluatorConfig, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type RubricItem, type RubricsEvaluatorConfig, type RunEvalCaseOptions, type RunEvaluationOptions, type ScoreRange, type ScriptExecutionContext, SimpleTraceFileExporter, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetAccessConfig, type TargetDefinition, TemplateNotDirectoryError, TemplateNotFoundError, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, TokenUsageEvaluator, type TokenUsageEvaluatorConfig, type TokenUsageEvaluatorOptions, type ToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceComputeResult, type TraceSummary, type TrialAggregation, type TrialResult, type TrialStrategy, type TrialsConfig, type UserTestMessage, type VSCodeResolvedConfig, type WorkspaceConfig, WorkspaceCreationError, type WorkspaceScriptConfig, assembleLlmJudgePrompt, avgToolDurationMs, buildDirectoryChain, buildOutputSchema, buildPromptInputs, buildRubricOutputSchema, buildScoreRangeOutputSchema, buildSearchRoots, calculateRubricScore, captureFileChanges, clampScore, cleanupEvalWorkspaces, cleanupWorkspace, computeTraceSummary, consumeClaudeLogEntries, consumeCodexLogEntries, consumeCopilotCliLogEntries, consumeCopilotSdkLogEntries, consumePiLogEntries, createAgentKernel, createBuiltinProviderRegistry, createBuiltinRegistry, createProvider, createTempWorkspace, deepEqual, defineConfig, detectFormat, discoverAssertions, discoverProviders, ensureVSCodeSubagents, evaluate, executeScript, executeWorkspaceScript, explorationRatio, extractCacheConfig, extractJsonBlob, extractTargetFromSuite, extractTargetsFromSuite, extractTargetsFromTestCase, extractTrialsConfig, fileExists, findGitRoot, freeformEvaluationSchema, generateRubrics, getHitCount, getWorkspacePath, initializeBaseline, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isNonEmptyString, isTestMessage, isTestMessageRole, listTargetNames, loadConfig, loadEvalCaseById, loadEvalCases, loadEvalSuite, loadTestById, loadTestSuite, loadTests, loadTsConfig, mergeExecutionMetrics, negateScore, normalizeLineEndings, parseJsonFromText, parseJsonSafe, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, resolveWorkspaceTemplate, rubricEvaluationSchema, runContainsAssertion, runEqualsAssertion, runEvalCase, runEvaluation, runIsJsonAssertion, runRegexAssertion, scoreToVerdict, shouldEnableCache, shouldSkipCacheForTemperature, subscribeToClaudeLogEntries, subscribeToCodexLogEntries, subscribeToCopilotCliLogEntries, subscribeToCopilotSdkLogEntries, subscribeToPiLogEntries, substituteVariables, toCamelCaseDeep, toSnakeCaseDeep, tokensPerTool, trimBaselineResult };

package/dist/index.d.ts

@@ -187,8 +187,6 @@ interface TargetDefinition {
     readonly subagentRoot?: string | unknown | undefined;
     readonly workspace_template?: string | unknown | undefined;
     readonly workspaceTemplate?: string | unknown | undefined;
-    readonly command_template?: string | unknown | undefined;
-    readonly commandTemplate?: string | unknown | undefined;
     readonly files_format?: string | unknown | undefined;
     readonly filesFormat?: string | unknown | undefined;
     readonly attachments_format?: string | unknown | undefined;
@@ -241,20 +239,22 @@ interface TraceSummary {
     readonly toolCallsByName: Readonly<Record<string, number>>;
     /** Number of error events */
     readonly errorCount: number;
-    /** Token usage metrics (optional, from provider) */
+    /** Per-tool duration arrays in milliseconds (optional) */
+    readonly toolDurations?: Readonly<Record<string, readonly number[]>>;
+    /** Number of LLM calls (assistant messages) */
+    readonly llmCallCount?: number;
+}
+/**
+ * Combined result of trace computation + execution metrics merge.
+ * Returned by computeTraceSummaryWithMetrics().
+ */
+interface TraceComputeResult {
+    readonly trace: TraceSummary;
     readonly tokenUsage?: TokenUsage;
-    /** Total cost in USD (optional, from provider) */
     readonly costUsd?: number;
-    /** Total execution duration in milliseconds (optional) */
     readonly durationMs?: number;
-    /** Per-tool duration arrays in milliseconds (optional) */
-    readonly toolDurations?: Readonly<Record<string, readonly number[]>>;
-    /** ISO 8601 timestamp when execution started (derived from earliest span) */
     readonly startTime?: string;
-    /** ISO 8601 timestamp when execution ended (derived from latest span) */
     readonly endTime?: string;
-    /** Number of LLM calls (assistant messages) */
-    readonly llmCallCount?: number;
 }
 /**
  * Argument matching mode for tool_trajectory expected items.
@@ -321,7 +321,7 @@ interface MessageLike {
  * - toolDurations: per-tool duration arrays (from durationMs or computed from start/end)
  * - llmCallCount: count of assistant messages
  */
-declare function computeTraceSummary(messages: readonly MessageLike[]): TraceSummary;
+declare function computeTraceSummary(messages: readonly MessageLike[]): TraceComputeResult;
 /**
  * Default tool names considered as exploration/read-only operations.
  * Can be overridden per-evaluation via config.
@@ -343,7 +343,7 @@ declare function explorationRatio(summary: TraceSummary, explorationTools?: read
  * @param summary - Trace summary with optional token usage
  * @returns Average tokens per tool call, or undefined
  */
-declare function tokensPerTool(summary: TraceSummary): number | undefined;
+declare function tokensPerTool(summary: TraceSummary, tokenUsage?: TokenUsage): number | undefined;
 /**
  * Average tool duration across all tool calls.
  * Returns undefined if toolDurations is not available or empty.
@@ -365,15 +365,15 @@ interface ExecutionMetrics {
     readonly endTime?: string;
 }
 /**
- * Merge execution metrics from provider response into a trace summary.
- * Returns a new TraceSummary with metrics fields populated.
+ * Merge execution metrics from provider response into a trace compute result.
+ * Returns a new TraceComputeResult with metrics fields populated.
  * Provider-level timing takes precedence over span-derived timing.
  *
- * @param summary - Base trace summary from computeTraceSummary
+ * @param computed - Base trace compute result from computeTraceSummary
  * @param metrics - Optional execution metrics from provider
- * @returns TraceSummary with merged metrics
+ * @returns TraceComputeResult with merged metrics
  */
-declare function mergeExecutionMetrics(summary: TraceSummary, metrics?: ExecutionMetrics): TraceSummary;
+declare function mergeExecutionMetrics(computed: TraceComputeResult, metrics?: ExecutionMetrics): TraceComputeResult;
 
 /**
  * JSON primitive values appearing in AgentV payloads.
@@ -924,6 +924,16 @@ interface EvaluationResult {
     readonly answer: string;
     readonly target: string;
     readonly reasoning?: string;
+    /** Token usage metrics from provider (optional) */
+    readonly tokenUsage?: TokenUsage;
+    /** Total cost in USD (optional, from provider) */
+    readonly costUsd?: number;
+    /** Total execution duration in milliseconds (optional) */
+    readonly durationMs?: number;
+    /** ISO 8601 timestamp when execution started */
+    readonly startTime?: string;
+    /** ISO 8601 timestamp when execution ended */
+    readonly endTime?: string;
     readonly requests?: {
         readonly agent?: JsonObject;
         readonly lm?: JsonObject;
@@ -955,6 +965,8 @@ interface EvaluationResult {
     readonly aggregation?: TrialAggregation;
     /** Whether the trial loop was terminated early due to cost limit */
     readonly costLimited?: boolean;
+    /** Whether the evaluation was skipped due to suite-level budget exhaustion */
+    readonly budgetExceeded?: boolean;
 }
 type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
 interface EvaluatorResult {
@@ -971,6 +983,8 @@ interface EvaluatorResult {
     readonly scores?: readonly EvaluatorResult[];
     /** Optional structured details from code judges (e.g., TP/TN/FP/FN counts). */
     readonly details?: JsonObject;
+    /** Token usage from LLM calls made by this evaluator (optional). */
+    readonly tokenUsage?: TokenUsage;
 }
 /**
  * Convenience accessor matching the Python hit_count property.
@@ -1116,6 +1130,8 @@ type EvalSuiteResult = {
     readonly cacheConfig?: CacheConfig;
     /** Suite-level metadata (name, description, version, etc.) */
     readonly metadata?: EvalMetadata;
+    /** Suite-level total cost budget in USD */
+    readonly totalBudgetUsd?: number;
 };
 /**
  * Load tests and suite metadata from a single parse.
@@ -1637,6 +1653,16 @@ interface EvaluationContext {
     readonly output?: readonly Message[];
     /** Lightweight summary of trace events (if available) */
     readonly trace?: TraceSummary;
+    /** Token usage from provider execution (promoted from TraceSummary) */
+    readonly tokenUsage?: TokenUsage;
+    /** Total cost in USD (from provider) */
+    readonly costUsd?: number;
+    /** Execution duration in milliseconds */
+    readonly durationMs?: number;
+    /** ISO 8601 timestamp when execution started */
+    readonly startTime?: string;
+    /** ISO 8601 timestamp when execution ended */
+    readonly endTime?: string;
     /** Resolver for target override in code judges */
     readonly targetResolver?: TargetResolver;
     /** List of available target names for code judges */
@@ -1657,6 +1683,8 @@ interface EvaluationScore {
     readonly scores?: readonly ChildEvaluatorResult[];
     /** Optional structured details from code judges (e.g., TP/TN/FP/FN counts, alignments). */
     readonly details?: JsonObject;
+    /** Token usage from LLM calls made by this evaluator (optional). */
+    readonly tokenUsage?: TokenUsage;
 }
 interface ChildEvaluatorResult {
     readonly name: string;
@@ -1671,6 +1699,8 @@ interface ChildEvaluatorResult {
     readonly scores?: readonly ChildEvaluatorResult[];
     /** Optional structured details from code judges (e.g., TP/TN/FP/FN counts, alignments). */
     readonly details?: JsonObject;
+    /** Token usage from LLM calls made by this evaluator (optional). */
+    readonly tokenUsage?: TokenUsage;
 }
 interface Evaluator {
     readonly kind: string;
@@ -1740,7 +1770,7 @@ interface CostEvaluatorOptions {
 }
 /**
  * Evaluator that checks execution cost against a budget.
- * Uses trace.costUsd from the evaluation context.
+ * Uses costUsd from the evaluation context.
  */
 declare class CostEvaluator implements Evaluator {
     readonly kind = "cost";
@@ -1812,7 +1842,7 @@ interface LatencyEvaluatorOptions {
 }
 /**
  * Evaluator that checks execution duration against a threshold.
- * Uses trace.durationMs from the evaluation context.
+ * Uses durationMs from the evaluation context.
  */
 declare class LatencyEvaluator implements Evaluator {
     readonly kind = "latency";
@@ -1987,7 +2017,7 @@ interface TokenUsageEvaluatorOptions {
 }
 /**
  * Evaluator that checks provider-reported token usage against configured limits.
- * Uses trace.tokenUsage from the evaluation context.
+ * Uses tokenUsage from the evaluation context.
  */
 declare class TokenUsageEvaluator implements Evaluator {
     readonly kind = "token_usage";
@@ -2196,6 +2226,8 @@ interface RunEvaluationOptions {
     readonly trials?: TrialsConfig;
     /** Real-time observability callbacks passed to the provider */
     readonly streamCallbacks?: ProviderStreamCallbacks;
+    /** Suite-level total cost budget in USD (stops dispatching when exceeded) */
+    readonly totalBudgetUsd?: number;
 }
 declare function runEvaluation(options: RunEvaluationOptions): Promise<readonly EvaluationResult[]>;
 declare function runEvalCase(options: RunEvalCaseOptions): Promise<EvaluationResult>;
@@ -2886,4 +2918,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { AgentJudgeEvaluator, type AgentJudgeEvaluatorConfig, type AgentJudgeEvaluatorOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVConfig as AgentVTsConfig, type AnthropicResolvedConfig, type ArgsMatchMode, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, type CacheConfig, type ChildEvaluatorResult, type ClaudeResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type ConfidenceIntervalAggregation, type ContainsEvaluatorConfig, type CopilotCliResolvedConfig, type CopilotSdkResolvedConfig, CostEvaluator, type CostEvaluatorConfig, type CostEvaluatorOptions, DEFAULT_EVALUATOR_TEMPLATE, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DeterministicAssertionEvaluator, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsEvaluatorConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTest, type EvalTestInput, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorDispatchContext, type EvaluatorFactory, type EvaluatorFactoryFn, type EvaluatorKind, EvaluatorRegistry, type EvaluatorResult, type ExecutionMetrics, ExecutionMetricsEvaluator, type ExecutionMetricsEvaluatorConfig, type ExecutionMetricsEvaluatorOptions, FieldAccuracyEvaluator, type FieldAccuracyEvaluatorConfig, type FieldAccuracyEvaluatorOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type IsJsonEvaluatorConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyEvaluator, type LatencyEvaluatorConfig, type LatencyEvaluatorOptions, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type LlmJudgePromptAssembly, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, type PassAtKAggregation, type PiAgentSdkResolvedConfig, type PiCodingAgentResolvedConfig, type ProgressEvent, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexEvaluatorConfig, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type RubricItem, type RubricsEvaluatorConfig, type RunEvalCaseOptions, type RunEvaluationOptions, type ScoreRange, type ScriptExecutionContext, SimpleTraceFileExporter, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetAccessConfig, type TargetDefinition, TemplateNotDirectoryError, TemplateNotFoundError, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, TokenUsageEvaluator, type TokenUsageEvaluatorConfig, type TokenUsageEvaluatorOptions, type ToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceSummary, type TrialAggregation, type TrialResult, type TrialStrategy, type TrialsConfig, type UserTestMessage, type VSCodeResolvedConfig, type WorkspaceConfig, WorkspaceCreationError, type WorkspaceScriptConfig, assembleLlmJudgePrompt, avgToolDurationMs, buildDirectoryChain, buildOutputSchema, buildPromptInputs, buildRubricOutputSchema, buildScoreRangeOutputSchema, buildSearchRoots, calculateRubricScore, captureFileChanges, clampScore, cleanupEvalWorkspaces, cleanupWorkspace, computeTraceSummary, consumeClaudeLogEntries, consumeCodexLogEntries, consumeCopilotCliLogEntries, consumeCopilotSdkLogEntries, consumePiLogEntries, createAgentKernel, createBuiltinProviderRegistry, createBuiltinRegistry, createProvider, createTempWorkspace, deepEqual, defineConfig, detectFormat, discoverAssertions, discoverProviders, ensureVSCodeSubagents, evaluate, executeScript, executeWorkspaceScript, explorationRatio, extractCacheConfig, extractJsonBlob, extractTargetFromSuite, extractTargetsFromSuite, extractTargetsFromTestCase, extractTrialsConfig, fileExists, findGitRoot, freeformEvaluationSchema, generateRubrics, getHitCount, getWorkspacePath, initializeBaseline, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isNonEmptyString, isTestMessage, isTestMessageRole, listTargetNames, loadConfig, loadEvalCaseById, loadEvalCases, loadEvalSuite, loadTestById, loadTestSuite, loadTests, loadTsConfig, mergeExecutionMetrics, negateScore, normalizeLineEndings, parseJsonFromText, parseJsonSafe, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, resolveWorkspaceTemplate, rubricEvaluationSchema, runContainsAssertion, runEqualsAssertion, runEvalCase, runEvaluation, runIsJsonAssertion, runRegexAssertion, scoreToVerdict, shouldEnableCache, shouldSkipCacheForTemperature, subscribeToClaudeLogEntries, subscribeToCodexLogEntries, subscribeToCopilotCliLogEntries, subscribeToCopilotSdkLogEntries, subscribeToPiLogEntries, substituteVariables, toCamelCaseDeep, toSnakeCaseDeep, tokensPerTool, trimBaselineResult };
+export { AgentJudgeEvaluator, type AgentJudgeEvaluatorConfig, type AgentJudgeEvaluatorOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVConfig as AgentVTsConfig, type AnthropicResolvedConfig, type ArgsMatchMode, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, type CacheConfig, type ChildEvaluatorResult, type ClaudeResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type ConfidenceIntervalAggregation, type ContainsEvaluatorConfig, type CopilotCliResolvedConfig, type CopilotSdkResolvedConfig, CostEvaluator, type CostEvaluatorConfig, type CostEvaluatorOptions, DEFAULT_EVALUATOR_TEMPLATE, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DeterministicAssertionEvaluator, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsEvaluatorConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTest, type EvalTestInput, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorDispatchContext, type EvaluatorFactory, type EvaluatorFactoryFn, type EvaluatorKind, EvaluatorRegistry, type EvaluatorResult, type ExecutionMetrics, ExecutionMetricsEvaluator, type ExecutionMetricsEvaluatorConfig, type ExecutionMetricsEvaluatorOptions, FieldAccuracyEvaluator, type FieldAccuracyEvaluatorConfig, type FieldAccuracyEvaluatorOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type IsJsonEvaluatorConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyEvaluator, type LatencyEvaluatorConfig, type LatencyEvaluatorOptions, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type LlmJudgePromptAssembly, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, type PassAtKAggregation, type PiAgentSdkResolvedConfig, type PiCodingAgentResolvedConfig, type ProgressEvent, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexEvaluatorConfig, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type RubricItem, type RubricsEvaluatorConfig, type RunEvalCaseOptions, type RunEvaluationOptions, type ScoreRange, type ScriptExecutionContext, SimpleTraceFileExporter, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetAccessConfig, type TargetDefinition, TemplateNotDirectoryError, TemplateNotFoundError, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, TokenUsageEvaluator, type TokenUsageEvaluatorConfig, type TokenUsageEvaluatorOptions, type ToolCall, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceComputeResult, type TraceSummary, type TrialAggregation, type TrialResult, type TrialStrategy, type TrialsConfig, type UserTestMessage, type VSCodeResolvedConfig, type WorkspaceConfig, WorkspaceCreationError, type WorkspaceScriptConfig, assembleLlmJudgePrompt, avgToolDurationMs, buildDirectoryChain, buildOutputSchema, buildPromptInputs, buildRubricOutputSchema, buildScoreRangeOutputSchema, buildSearchRoots, calculateRubricScore, captureFileChanges, clampScore, cleanupEvalWorkspaces, cleanupWorkspace, computeTraceSummary, consumeClaudeLogEntries, consumeCodexLogEntries, consumeCopilotCliLogEntries, consumeCopilotSdkLogEntries, consumePiLogEntries, createAgentKernel, createBuiltinProviderRegistry, createBuiltinRegistry, createProvider, createTempWorkspace, deepEqual, defineConfig, detectFormat, discoverAssertions, discoverProviders, ensureVSCodeSubagents, evaluate, executeScript, executeWorkspaceScript, explorationRatio, extractCacheConfig, extractJsonBlob, extractTargetFromSuite, extractTargetsFromSuite, extractTargetsFromTestCase, extractTrialsConfig, fileExists, findGitRoot, freeformEvaluationSchema, generateRubrics, getHitCount, getWorkspacePath, initializeBaseline, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isNonEmptyString, isTestMessage, isTestMessageRole, listTargetNames, loadConfig, loadEvalCaseById, loadEvalCases, loadEvalSuite, loadTestById, loadTestSuite, loadTests, loadTsConfig, mergeExecutionMetrics, negateScore, normalizeLineEndings, parseJsonFromText, parseJsonSafe, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, resolveWorkspaceTemplate, rubricEvaluationSchema, runContainsAssertion, runEqualsAssertion, runEvalCase, runEvaluation, runIsJsonAssertion, runRegexAssertion, scoreToVerdict, shouldEnableCache, shouldSkipCacheForTemperature, subscribeToClaudeLogEntries, subscribeToCodexLogEntries, subscribeToCopilotCliLogEntries, subscribeToCopilotSdkLogEntries, subscribeToPiLogEntries, substituteVariables, toCamelCaseDeep, toSnakeCaseDeep, tokensPerTool, trimBaselineResult };