@agentv/core 1.4.0 → 2.0.1

package/dist/index.d.cts

@@ -1,9 +1,21 @@
+import { z } from 'zod';
 import * as ai from 'ai';
 
 /**
  * Trace event types for capturing agent execution traces.
  * Provides a normalized, provider-agnostic model for tool-call trajectories.
  */
+/**
+ * Token usage metrics from provider execution.
+ */
+interface TokenUsage {
+    /** Input/prompt tokens consumed */
+    readonly input: number;
+    /** Output/completion tokens generated */
+    readonly output: number;
+    /** Cached tokens (optional, provider-specific) */
+    readonly cached?: number;
+}
 /**
  * Compact summary of a trace for lightweight persistence.
  * Included in results by default to avoid payload bloat.
@@ -17,6 +29,14 @@ interface TraceSummary {
     readonly toolCallsByName: Readonly<Record<string, number>>;
     /** Number of error events */
     readonly errorCount: number;
+    /** Token usage metrics (optional, from provider) */
+    readonly tokenUsage?: TokenUsage;
+    /** Total cost in USD (optional, from provider) */
+    readonly costUsd?: number;
+    /** Total execution duration in milliseconds (optional) */
+    readonly durationMs?: number;
+    /** Per-tool duration arrays in milliseconds (optional) */
+    readonly toolDurations?: Readonly<Record<string, readonly number[]>>;
 }
 /**
  * Configuration for tool_trajectory evaluator.
@@ -38,6 +58,8 @@ interface ToolTrajectoryEvaluatorConfig {
  */
 interface ToolTrajectoryExpectedItem {
     readonly tool: string;
+    /** Optional argument matching: 'any' skips validation, object performs partial deep equality */
+    readonly args?: 'any' | Record<string, unknown>;
 }
 /**
  * Simplified input type for computeTraceSummary.
@@ -53,6 +75,53 @@ interface OutputMessageLike {
  * Used for default result persistence without payload bloat.
  */
 declare function computeTraceSummary(messages: readonly OutputMessageLike[]): TraceSummary;
+/**
+ * Default tool names considered as exploration/read-only operations.
+ * Can be overridden per-evaluation via config.
+ */
+declare const DEFAULT_EXPLORATION_TOOLS: readonly ["read", "grep", "glob", "search", "list", "Read", "Grep", "Glob", "WebSearch", "WebFetch"];
+/**
+ * Ratio of exploration tool calls to total tool calls.
+ * Returns undefined if there are no tool calls.
+ *
+ * @param summary - Trace summary with tool call counts
+ * @param explorationTools - Tool names considered exploration (defaults to DEFAULT_EXPLORATION_TOOLS)
+ * @returns Ratio between 0 and 1, or undefined if no tool calls
+ */
+declare function explorationRatio(summary: TraceSummary, explorationTools?: readonly string[]): number | undefined;
+/**
+ * Average tokens consumed per tool call.
+ * Returns undefined if tokenUsage is not available or no tool calls.
+ *
+ * @param summary - Trace summary with optional token usage
+ * @returns Average tokens per tool call, or undefined
+ */
+declare function tokensPerTool(summary: TraceSummary): number | undefined;
+/**
+ * Average tool duration across all tool calls.
+ * Returns undefined if toolDurations is not available or empty.
+ *
+ * @param summary - Trace summary with optional tool durations
+ * @returns Average duration in milliseconds, or undefined
+ */
+declare function avgToolDurationMs(summary: TraceSummary): number | undefined;
+/**
+ * Execution metrics from provider response.
+ */
+interface ExecutionMetrics {
+    readonly tokenUsage?: TokenUsage;
+    readonly costUsd?: number;
+    readonly durationMs?: number;
+}
+/**
+ * Merge execution metrics from provider response into a trace summary.
+ * Returns a new TraceSummary with metrics fields populated.
+ *
+ * @param summary - Base trace summary from computeTraceSummary
+ * @param metrics - Optional execution metrics from provider
+ * @returns TraceSummary with merged metrics
+ */
+declare function mergeExecutionMetrics(summary: TraceSummary, metrics?: ExecutionMetrics): TraceSummary;
 
 /**
  * JSON primitive values appearing in AgentV payloads.
@@ -132,17 +201,19 @@ declare function isJsonValue(value: unknown): value is JsonValue;
  * - Either content (string or array of objects) OR tool_calls (for assistant messages)
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite", "tool_trajectory"];
+declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite", "tool_trajectory", "field_accuracy", "latency", "cost", "token_usage"];
 type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 type CodeEvaluatorConfig = {
     readonly name: string;
     readonly type: 'code';
-    readonly script: string;
+    readonly script: readonly string[];
     readonly resolvedScriptPath?: string;
     readonly cwd?: string;
     readonly resolvedCwd?: string;
     readonly weight?: number;
+    /** Pass-through configuration for the code_judge script (any unrecognized YAML properties) */
+    readonly config?: JsonObject;
 };
 type LlmJudgeEvaluatorConfig = {
     readonly name: string;
@@ -178,7 +249,85 @@ type CompositeEvaluatorConfig = {
     readonly aggregator: CompositeAggregatorConfig;
     readonly weight?: number;
 };
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig | ToolTrajectoryEvaluatorConfig;
+/**
+ * Match type for field accuracy evaluation.
+ * Note: For fuzzy string matching (Levenshtein, Jaro-Winkler, etc.), use a code_judge evaluator.
+ * See examples/features/document-extraction/fuzzy_match.ts for an example.
+ */
+type FieldMatchType = 'exact' | 'numeric_tolerance' | 'date';
+/**
+ * Aggregation strategy for combining field scores.
+ */
+type FieldAggregationType = 'weighted_average' | 'all_or_nothing';
+/**
+ * Configuration for a single field to evaluate.
+ */
+type FieldConfig = {
+    /** Dot-notation path to the field (e.g., "invoice.vendor.name" or "items[0].amount") */
+    readonly path: string;
+    /** Match strategy for this field */
+    readonly match: FieldMatchType;
+    /** Whether this field is required (missing required fields count as failures) */
+    readonly required?: boolean;
+    /** Weight for aggregation (default: 1.0) */
+    readonly weight?: number;
+    /** Tolerance for numeric matching (absolute value unless relative is true) */
+    readonly tolerance?: number;
+    /** Whether tolerance is relative (percentage) vs absolute */
+    readonly relative?: boolean;
+    /** Date formats to try when parsing (default: common formats) */
+    readonly formats?: readonly string[];
+};
+/**
+ * Configuration for the field_accuracy evaluator.
+ */
+type FieldAccuracyEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'field_accuracy';
+    /** Fields to compare between candidate and expected */
+    readonly fields: readonly FieldConfig[];
+    /** Strategy for combining field scores (default: weighted_average) */
+    readonly aggregation?: FieldAggregationType;
+    readonly weight?: number;
+};
+/**
+ * Configuration for the latency evaluator.
+ * Checks execution duration against a threshold.
+ */
+type LatencyEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'latency';
+    /** Maximum allowed duration in milliseconds */
+    readonly threshold: number;
+    readonly weight?: number;
+};
+/**
+ * Configuration for the cost evaluator.
+ * Checks execution cost against a budget.
+ */
+type CostEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'cost';
+    /** Maximum allowed cost in USD */
+    readonly budget: number;
+    readonly weight?: number;
+};
+/**
+ * Configuration for the token_usage evaluator.
+ * Checks provider-reported token usage against configured limits.
+ */
+type TokenUsageEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'token_usage';
+    /** Maximum allowed total tokens (input + output + cached, when present) */
+    readonly max_total?: number;
+    /** Maximum allowed input tokens (prompt) */
+    readonly max_input?: number;
+    /** Maximum allowed output tokens (completion) */
+    readonly max_output?: number;
+    readonly weight?: number;
+};
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig | ToolTrajectoryEvaluatorConfig | FieldAccuracyEvaluatorConfig | LatencyEvaluatorConfig | CostEvaluatorConfig | TokenUsageEvaluatorConfig;
 /**
  * Eval case definition sourced from AgentV specs.
  */
@@ -204,23 +353,22 @@ interface EvalCase {
  */
 interface EvaluationResult {
     readonly timestamp: string;
-    readonly eval_id: string;
+    readonly evalId: string;
     readonly dataset?: string;
-    readonly conversation_id?: string;
+    readonly conversationId?: string;
     readonly score: number;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
-    readonly candidate_answer: string;
+    readonly candidateAnswer: string;
     readonly target: string;
     readonly reasoning?: string;
-    readonly raw_aspects?: readonly string[];
-    readonly agent_provider_request?: JsonObject;
-    readonly lm_provider_request?: JsonObject;
-    readonly evaluator_provider_request?: JsonObject;
-    readonly evaluator_results?: readonly EvaluatorResult[];
+    readonly agentProviderRequest?: JsonObject;
+    readonly lmProviderRequest?: JsonObject;
+    readonly evaluatorProviderRequest?: JsonObject;
+    readonly evaluatorResults?: readonly EvaluatorResult[];
     readonly error?: string;
     /** Lightweight summary of the execution trace (always included when available) */
-    readonly trace_summary?: TraceSummary;
+    readonly traceSummary?: TraceSummary;
 }
 type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
 interface EvaluatorResult {
@@ -232,9 +380,9 @@ interface EvaluatorResult {
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly reasoning?: string;
-    readonly raw_request?: JsonObject;
-    readonly evaluator_provider_request?: JsonObject;
-    readonly evaluator_results?: readonly EvaluatorResult[];
+    readonly rawRequest?: JsonObject;
+    readonly evaluatorProviderRequest?: JsonObject;
+    readonly evaluatorResults?: readonly EvaluatorResult[];
 }
 /**
  * Convenience accessor matching the Python hit_count property.
@@ -248,7 +396,7 @@ interface ChatMessage {
     readonly name?: string;
 }
 type ChatPrompt = readonly ChatMessage[];
-type ProviderKind = 'azure' | 'anthropic' | 'gemini' | 'codex' | 'cli' | 'mock' | 'vscode' | 'vscode-insiders';
+type ProviderKind = 'azure' | 'anthropic' | 'gemini' | 'codex' | 'pi-coding-agent' | 'claude-code' | 'cli' | 'mock' | 'vscode' | 'vscode-insiders';
 interface ProviderRequest {
     readonly question: string;
     readonly systemPrompt?: string;
@@ -297,11 +445,28 @@ interface OutputMessage {
     /** Provider-specific metadata */
     readonly metadata?: Record<string, unknown>;
 }
+/**
+ * Token usage metrics reported by provider.
+ */
+interface ProviderTokenUsage {
+    /** Input/prompt tokens consumed */
+    readonly input: number;
+    /** Output/completion tokens generated */
+    readonly output: number;
+    /** Cached tokens (optional, provider-specific) */
+    readonly cached?: number;
+}
 interface ProviderResponse {
     readonly raw?: unknown;
     readonly usage?: JsonObject;
     /** Output messages from agent execution (primary source for tool trajectory) */
     readonly outputMessages?: readonly OutputMessage[];
+    /** Token usage metrics (optional) */
+    readonly tokenUsage?: ProviderTokenUsage;
+    /** Total cost in USD (optional) */
+    readonly costUsd?: number;
+    /** Execution duration in milliseconds (optional) */
+    readonly durationMs?: number;
 }
 interface Provider {
     readonly id: string;
@@ -482,6 +647,101 @@ declare function resolveFileReference(rawValue: string, searchRoots: readonly st
     readonly attempted: readonly string[];
 }>;
 
+/**
+ * Strict normalized schema for CLI target configuration.
+ * This is the final validated shape after environment variable resolution
+ * and snake_case to camelCase normalization.
+ *
+ * Uses .strict() to reject unknown properties, ensuring configuration
+ * errors are caught early rather than silently ignored.
+ *
+ * @example
+ * ```typescript
+ * const config: CliNormalizedConfig = {
+ *   commandTemplate: 'agent run {PROMPT}',
+ *   timeoutMs: 120000,
+ *   verbose: true,
+ * };
+ * CliTargetConfigSchema.parse(config); // Validates the normalized config
+ * ```
+ */
+declare const CliTargetConfigSchema: z.ZodObject<{
+    commandTemplate: z.ZodString;
+    filesFormat: z.ZodOptional<z.ZodString>;
+    cwd: z.ZodOptional<z.ZodString>;
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
+    healthcheck: z.ZodOptional<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+        type: z.ZodLiteral<"http">;
+        url: z.ZodString;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, "strict", z.ZodTypeAny, {
+        type: "http";
+        url: string;
+        timeoutMs?: number | undefined;
+    }, {
+        type: "http";
+        url: string;
+        timeoutMs?: number | undefined;
+    }>, z.ZodObject<{
+        type: z.ZodLiteral<"command">;
+        commandTemplate: z.ZodString;
+        cwd: z.ZodOptional<z.ZodString>;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, "strict", z.ZodTypeAny, {
+        type: "command";
+        commandTemplate: string;
+        cwd?: string | undefined;
+        timeoutMs?: number | undefined;
+    }, {
+        type: "command";
+        commandTemplate: string;
+        cwd?: string | undefined;
+        timeoutMs?: number | undefined;
+    }>]>>;
+    verbose: z.ZodOptional<z.ZodBoolean>;
+    keepTempFiles: z.ZodOptional<z.ZodBoolean>;
+}, "strict", z.ZodTypeAny, {
+    commandTemplate: string;
+    cwd?: string | undefined;
+    verbose?: boolean | undefined;
+    filesFormat?: string | undefined;
+    healthcheck?: {
+        type: "http";
+        url: string;
+        timeoutMs?: number | undefined;
+    } | {
+        type: "command";
+        commandTemplate: string;
+        cwd?: string | undefined;
+        timeoutMs?: number | undefined;
+    } | undefined;
+    keepTempFiles?: boolean | undefined;
+    timeoutMs?: number | undefined;
+}, {
+    commandTemplate: string;
+    cwd?: string | undefined;
+    verbose?: boolean | undefined;
+    filesFormat?: string | undefined;
+    healthcheck?: {
+        type: "http";
+        url: string;
+        timeoutMs?: number | undefined;
+    } | {
+        type: "command";
+        commandTemplate: string;
+        cwd?: string | undefined;
+        timeoutMs?: number | undefined;
+    } | undefined;
+    keepTempFiles?: boolean | undefined;
+    timeoutMs?: number | undefined;
+}>;
+type CliNormalizedConfig = z.infer<typeof CliTargetConfigSchema>;
+/**
+ * Resolved CLI configuration type derived from CliTargetConfigSchema.
+ * This is the final validated shape used by the CLI provider at runtime.
+ * Using Readonly to ensure immutability for runtime safety.
+ */
+type CliResolvedConfig = Readonly<CliNormalizedConfig>;
 interface RetryConfig {
     readonly maxRetries?: number;
     readonly initialDelayMs?: number;
@@ -529,6 +789,31 @@ interface CodexResolvedConfig {
     readonly timeoutMs?: number;
     readonly logDir?: string;
     readonly logFormat?: 'summary' | 'json';
+    readonly systemPrompt?: string;
+}
+interface PiCodingAgentResolvedConfig {
+    readonly executable: string;
+    readonly provider?: string;
+    readonly model?: string;
+    readonly apiKey?: string;
+    readonly tools?: string;
+    readonly thinking?: string;
+    readonly args?: readonly string[];
+    readonly cwd?: string;
+    readonly timeoutMs?: number;
+    readonly logDir?: string;
+    readonly logFormat?: 'summary' | 'json';
+    readonly systemPrompt?: string;
+}
+interface ClaudeCodeResolvedConfig {
+    readonly executable: string;
+    readonly model?: string;
+    readonly systemPrompt?: string;
+    readonly args?: readonly string[];
+    readonly cwd?: string;
+    readonly timeoutMs?: number;
+    readonly logDir?: string;
+    readonly logFormat?: 'summary' | 'json';
 }
 interface MockResolvedConfig {
     readonly response?: string;
@@ -543,25 +828,6 @@ interface VSCodeResolvedConfig {
     readonly subagentRoot?: string;
     readonly workspaceTemplate?: string;
 }
-type CliHealthcheck = {
-    readonly type: 'http';
-    readonly url: string;
-    readonly timeoutMs?: number;
-} | {
-    readonly type: 'command';
-    readonly commandTemplate: string;
-    readonly timeoutMs?: number;
-    readonly cwd?: string;
-};
-interface CliResolvedConfig {
-    readonly commandTemplate: string;
-    readonly filesFormat?: string;
-    readonly cwd?: string;
-    readonly timeoutMs?: number;
-    readonly healthcheck?: CliHealthcheck;
-    readonly verbose?: boolean;
-    readonly keepTempFiles?: boolean;
-}
 type ResolvedTarget = {
     readonly kind: 'azure';
     readonly name: string;
@@ -590,6 +856,20 @@ type ResolvedTarget = {
     readonly workers?: number;
     readonly providerBatching?: boolean;
     readonly config: CodexResolvedConfig;
+} | {
+    readonly kind: 'pi-coding-agent';
+    readonly name: string;
+    readonly judgeTarget?: string;
+    readonly workers?: number;
+    readonly providerBatching?: boolean;
+    readonly config: PiCodingAgentResolvedConfig;
+} | {
+    readonly kind: 'claude-code';
+    readonly name: string;
+    readonly judgeTarget?: string;
+    readonly workers?: number;
+    readonly providerBatching?: boolean;
+    readonly config: ClaudeCodeResolvedConfig;
 } | {
     readonly kind: 'mock';
     readonly name: string;
@@ -645,6 +925,26 @@ type CodexLogListener = (entry: CodexLogEntry) => void;
 declare function consumeCodexLogEntries(): CodexLogEntry[];
 declare function subscribeToCodexLogEntries(listener: CodexLogListener): () => void;
 
+type PiLogEntry = {
+    readonly filePath: string;
+    readonly evalCaseId?: string;
+    readonly targetName: string;
+    readonly attempt?: number;
+};
+type PiLogListener = (entry: PiLogEntry) => void;
+declare function consumePiLogEntries(): PiLogEntry[];
+declare function subscribeToPiLogEntries(listener: PiLogListener): () => void;
+
+type ClaudeCodeLogEntry = {
+    readonly filePath: string;
+    readonly evalCaseId?: string;
+    readonly targetName: string;
+    readonly attempt?: number;
+};
+type ClaudeCodeLogListener = (entry: ClaudeCodeLogEntry) => void;
+declare function consumeClaudeCodeLogEntries(): ClaudeCodeLogEntry[];
+declare function subscribeToClaudeCodeLogEntries(listener: ClaudeCodeLogListener): () => void;
+
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
@@ -676,7 +976,6 @@ interface EvaluationScore {
     readonly misses: readonly string[];
     readonly expectedAspectCount: number;
     readonly reasoning?: string;
-    readonly rawAspects?: readonly string[];
     readonly evaluatorRawRequest?: JsonObject;
     readonly evaluatorResults?: readonly ChildEvaluatorResult[];
 }
@@ -717,15 +1016,18 @@ declare class LlmJudgeEvaluator implements Evaluator {
     private runWithRetry;
 }
 interface CodeEvaluatorOptions {
-    readonly script: string;
+    readonly script: readonly string[];
     readonly cwd?: string;
     readonly agentTimeoutMs?: number;
+    /** Pass-through configuration from YAML (any unrecognized properties) */
+    readonly config?: Record<string, unknown>;
 }
 declare class CodeEvaluator implements Evaluator {
     readonly kind = "code";
     private readonly script;
     private readonly cwd?;
     private readonly agentTimeoutMs?;
+    private readonly config?;
     constructor(options: CodeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
 }
@@ -749,6 +1051,44 @@ declare class ToolTrajectoryEvaluator implements Evaluator {
     private evaluateInOrder;
     private evaluateExact;
 }
+interface FieldAccuracyEvaluatorOptions {
+    readonly config: FieldAccuracyEvaluatorConfig;
+}
+/**
+ * FieldAccuracyEvaluator compares extracted structured data against expected values
+ * with configurable matching strategies (exact, fuzzy, numeric_tolerance, date).
+ */
+declare class FieldAccuracyEvaluator implements Evaluator {
+    readonly kind = "field_accuracy";
+    private readonly config;
+    constructor(options: FieldAccuracyEvaluatorOptions);
+    evaluate(context: EvaluationContext): EvaluationScore;
+    /**
+     * Extract expected data from expected_messages array.
+     * Looks for the last assistant message with content.
+     */
+    private extractExpectedData;
+    /**
+     * Evaluate a single field against the expected value.
+     */
+    private evaluateField;
+    /**
+     * Exact equality comparison.
+     */
+    private compareExact;
+    /**
+     * Numeric comparison with absolute or relative tolerance.
+     */
+    private compareNumericTolerance;
+    /**
+     * Date comparison with format normalization.
+     */
+    private compareDate;
+    /**
+     * Aggregate field results using configured strategy.
+     */
+    private aggregateResults;
+}
 interface EvaluatorFactory {
     create(config: EvaluatorConfig, context: EvaluationContext): Evaluator;
 }
@@ -769,6 +1109,45 @@ declare class CompositeEvaluator implements Evaluator {
     private runCodeAggregator;
     private runLlmAggregator;
 }
+interface LatencyEvaluatorOptions {
+    readonly config: LatencyEvaluatorConfig;
+}
+/**
+ * Evaluator that checks execution duration against a threshold.
+ * Uses traceSummary.durationMs from the evaluation context.
+ */
+declare class LatencyEvaluator implements Evaluator {
+    readonly kind = "latency";
+    private readonly config;
+    constructor(options: LatencyEvaluatorOptions);
+    evaluate(context: EvaluationContext): EvaluationScore;
+}
+interface CostEvaluatorOptions {
+    readonly config: CostEvaluatorConfig;
+}
+/**
+ * Evaluator that checks execution cost against a budget.
+ * Uses traceSummary.costUsd from the evaluation context.
+ */
+declare class CostEvaluator implements Evaluator {
+    readonly kind = "cost";
+    private readonly config;
+    constructor(options: CostEvaluatorOptions);
+    evaluate(context: EvaluationContext): EvaluationScore;
+}
+interface TokenUsageEvaluatorOptions {
+    readonly config: TokenUsageEvaluatorConfig;
+}
+/**
+ * Evaluator that checks provider-reported token usage against configured limits.
+ * Uses traceSummary.tokenUsage from the evaluation context.
+ */
+declare class TokenUsageEvaluator implements Evaluator {
+    readonly kind = "token_usage";
+    private readonly config;
+    constructor(options: TokenUsageEvaluatorOptions);
+    evaluate(context: EvaluationContext): EvaluationScore;
+}
 
 type MaybePromise<T> = T | Promise<T>;
 interface EvaluationCache {
@@ -785,7 +1164,6 @@ interface RunEvalCaseOptions {
     readonly now?: () => Date;
     readonly maxRetries?: number;
     readonly agentTimeoutMs?: number;
-    readonly promptDumpDir?: string;
     readonly cache?: EvaluationCache;
     readonly useCache?: boolean;
     readonly signal?: AbortSignal;
@@ -809,7 +1187,6 @@ interface RunEvaluationOptions {
     readonly evaluators?: Partial<Record<string, Evaluator>>;
     readonly maxRetries?: number;
     readonly agentTimeoutMs?: number;
-    readonly promptDumpDir?: string;
     readonly cache?: EvaluationCache;
     readonly useCache?: boolean;
     readonly now?: () => Date;
@@ -834,9 +1211,37 @@ interface GenerateRubricsOptions {
  */
 declare function generateRubrics(options: GenerateRubricsOptions): Promise<readonly RubricItem[]>;
 
+/**
+ * Payload received by code judges via stdin.
+ * All properties use camelCase for TypeScript ergonomics.
+ */
+interface CodeJudgePayload {
+    readonly question: string;
+    readonly expectedOutcome: string;
+    readonly expectedMessages: readonly JsonObject[];
+    readonly referenceAnswer?: string;
+    readonly candidateAnswer: string;
+    readonly outputMessages?: readonly OutputMessage[] | null;
+    readonly guidelineFiles: readonly string[];
+    readonly inputFiles: readonly string[];
+    readonly inputMessages: readonly TestMessage[];
+    readonly traceSummary?: TraceSummary | null;
+    readonly config?: JsonObject | null;
+}
+/**
+ * Parse stdin JSON (snake_case) into typed camelCase object.
+ * Use this in TypeScript code judges to get type-safe, idiomatic input.
+ */
+declare function parseCodeJudgePayload(payload: string): CodeJudgePayload;
+/**
+ * Convenience helper that reads stdin and parses it.
+ * Equivalent to: parseCodeJudgePayload(readFileSync(0, 'utf8'))
+ */
+declare function readCodeJudgePayload(): CodeJudgePayload;
+
 type AgentKernel = {
     status: string;
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type ClaudeCodeResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CodeJudgePayload, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, CostEvaluator, type CostEvaluatorConfig, type CostEvaluatorOptions, DEFAULT_EXPLORATION_TOOLS, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type ExecutionMetrics, FieldAccuracyEvaluator, type FieldAccuracyEvaluatorConfig, type FieldAccuracyEvaluatorOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LatencyEvaluator, type LatencyEvaluatorConfig, type LatencyEvaluatorOptions, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type PiCodingAgentResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ProviderTokenUsage, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, TokenUsageEvaluator, type TokenUsageEvaluatorConfig, type TokenUsageEvaluatorOptions, type ToolTestMessage, ToolTrajectoryEvaluator, type ToolTrajectoryEvaluatorConfig, type ToolTrajectoryEvaluatorOptions, type ToolTrajectoryExpectedItem, type TraceSummary, type UserTestMessage, type VSCodeResolvedConfig, avgToolDurationMs, buildDirectoryChain, buildPromptInputs, buildSearchRoots, computeTraceSummary, consumeClaudeCodeLogEntries, consumeCodexLogEntries, consumePiLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, explorationRatio, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, mergeExecutionMetrics, normalizeLineEndings, parseCodeJudgePayload, readCodeJudgePayload, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToClaudeCodeLogEntries, subscribeToCodexLogEntries, subscribeToPiLogEntries, tokensPerTool };