@agentv/core 0.2.8 → 0.5.0

package/dist/index.d.cts

@@ -75,19 +75,25 @@ declare function isJsonValue(value: unknown): value is JsonValue;
  * Guard validating raw test messages.
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const GRADER_KIND_VALUES: readonly ["heuristic", "llm_judge"];
-/**
- * Supported grader implementations.
- */
-declare const GRADER_KINDS: readonly ["heuristic", "llm_judge"];
-/**
- * Grader identifiers available to the pipeline.
- */
-type GraderKind = (typeof GRADER_KIND_VALUES)[number];
-/**
- * Guard validating grader identifiers.
- */
-declare function isGraderKind(value: unknown): value is GraderKind;
+declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge"];
+type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
+declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
+type CodeEvaluatorConfig = {
+    readonly name: string;
+    readonly type: "code";
+    readonly script: string;
+    readonly resolvedScriptPath?: string;
+    readonly cwd?: string;
+    readonly resolvedCwd?: string;
+};
+type LlmJudgeEvaluatorConfig = {
+    readonly name: string;
+    readonly type: "llm_judge";
+    readonly prompt?: string;
+    readonly promptPath?: string;
+    readonly model?: string;
+};
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
 /**
  * Test case definition sourced from AgentV specs.
  */
@@ -103,7 +109,8 @@ interface EvalCase {
     readonly file_paths: readonly string[];
     readonly code_snippets: readonly string[];
     readonly outcome: string;
-    readonly grader: GraderKind;
+    readonly evaluator?: EvaluatorKind;
+    readonly evaluators?: readonly EvaluatorConfig[];
 }
 /**
  * Evaluator scorecard for a single test case run.
@@ -121,7 +128,18 @@ interface EvaluationResult {
     readonly reasoning?: string;
     readonly raw_aspects?: readonly string[];
     readonly raw_request?: JsonObject;
-    readonly grader_raw_request?: JsonObject;
+    readonly evaluator_raw_request?: JsonObject;
+    readonly evaluator_results?: readonly EvaluatorResult[];
+}
+interface EvaluatorResult {
+    readonly name: string;
+    readonly type: EvaluatorKind;
+    readonly score: number;
+    readonly hits: readonly string[];
+    readonly misses: readonly string[];
+    readonly reasoning?: string;
+    readonly raw_request?: JsonObject;
+    readonly evaluator_raw_request?: JsonObject;
 }
 /**
  * Convenience accessor matching the Python hit_count property.
@@ -177,13 +195,13 @@ declare function resolveFileReference(rawValue: string, searchRoots: readonly st
 }>;
 
 type ChatPrompt = AxChatRequest["chatPrompt"];
-type ProviderKind = "azure" | "anthropic" | "gemini" | "mock" | "vscode" | "vscode-insiders";
+type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
 interface ProviderRequest {
     readonly prompt: string;
     readonly guidelines?: string;
     readonly guideline_patterns?: readonly string[];
     readonly chatPrompt?: ChatPrompt;
-    readonly attachments?: readonly string[];
+    readonly inputFiles?: readonly string[];
     readonly evalCaseId?: string;
     readonly attempt?: number;
     readonly maxOutputTokens?: number;
@@ -202,6 +220,15 @@ interface Provider {
     readonly kind: ProviderKind;
     readonly targetName: string;
     invoke(request: ProviderRequest): Promise<ProviderResponse>;
+    /**
+     * Optional capability marker for provider-managed batching (single session handling multiple requests).
+     */
+    readonly supportsBatch?: boolean;
+    /**
+     * Optional batch invocation hook. When defined alongside supportsBatch=true,
+     * the orchestrator may send multiple requests in a single provider session.
+     */
+    invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -233,6 +260,12 @@ interface GeminiResolvedConfig {
     readonly temperature?: number;
     readonly maxOutputTokens?: number;
 }
+interface CodexResolvedConfig {
+    readonly executable: string;
+    readonly args?: readonly string[];
+    readonly cwd?: string;
+    readonly timeoutMs?: number;
+}
 interface MockResolvedConfig {
     readonly response?: string;
     readonly delayMs?: number;
@@ -246,36 +279,73 @@ interface VSCodeResolvedConfig {
     readonly subagentRoot?: string;
     readonly workspaceTemplate?: string;
 }
+type CliHealthcheck = {
+    readonly type: "http";
+    readonly url: string;
+    readonly timeoutMs?: number;
+} | {
+    readonly type: "command";
+    readonly commandTemplate: string;
+    readonly timeoutMs?: number;
+    readonly cwd?: string;
+};
+interface CliResolvedConfig {
+    readonly commandTemplate: string;
+    readonly filesFormat?: string;
+    readonly cwd?: string;
+    readonly env?: Record<string, string>;
+    readonly timeoutMs?: number;
+    readonly healthcheck?: CliHealthcheck;
+}
 type ResolvedTarget = {
     readonly kind: "azure";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: AzureResolvedConfig;
 } | {
     readonly kind: "anthropic";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: AnthropicResolvedConfig;
 } | {
     readonly kind: "gemini";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: GeminiResolvedConfig;
+} | {
+    readonly kind: "codex";
+    readonly name: string;
+    readonly judgeTarget?: string;
+    readonly workers?: number;
+    readonly providerBatching?: boolean;
+    readonly config: CodexResolvedConfig;
 } | {
     readonly kind: "mock";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: MockResolvedConfig;
 } | {
     readonly kind: "vscode" | "vscode-insiders";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: VSCodeResolvedConfig;
+} | {
+    readonly kind: "cli";
+    readonly name: string;
+    readonly judgeTarget?: string;
+    readonly workers?: number;
+    readonly providerBatching?: boolean;
+    readonly config: CliResolvedConfig;
 };
 declare function resolveTargetDefinition(definition: TargetDefinition, env?: EnvLookup): ResolvedTarget;
 
@@ -303,36 +373,7 @@ declare function ensureVSCodeSubagents(options: EnsureSubagentsOptions): Promise
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
-interface HeuristicScore {
-    readonly score: number;
-    readonly hits: readonly string[];
-    readonly misses: readonly string[];
-    readonly hitCount: number;
-    readonly totalAspects: number;
-    readonly rawAspects: readonly string[];
-}
-/**
- * Extract individual evaluation aspects from the expected assistant response.
- */
-declare function extractAspects(expectedResponse: string): readonly string[];
-/**
- * Determine which aspects were covered by the candidate response.
- */
-declare function calculateHits(candidateResponse: string, expectedAspects: readonly string[]): readonly string[];
-/**
- * Determine which aspects were not satisfied by the candidate response.
- */
-declare function calculateMisses(candidateResponse: string, expectedAspects: readonly string[], resolvedHits?: readonly string[]): readonly string[];
-/**
- * Evaluate the candidate response against the expected aspects.
- */
-declare function scoreCandidateResponse(candidateResponse: string, expectedAspects: readonly string[]): HeuristicScore;
-/**
- * Detect common error-prefixed outputs from providers.
- */
-declare function isErrorLike(text: string | undefined | null): boolean;
-
-interface GradeContext {
+interface EvaluationContext {
     readonly evalCase: EvalCase;
     readonly candidate: string;
     readonly target: ResolvedTarget;
@@ -341,40 +382,55 @@ interface GradeContext {
     readonly promptInputs: {
         readonly request: string;
         readonly guidelines: string;
+        readonly systemMessage?: string;
     };
     readonly now: Date;
     readonly judgeProvider?: Provider;
+    readonly systemPrompt?: string;
+    readonly evaluator?: EvaluatorConfig;
+    readonly judgeModel?: string;
 }
-interface GradeResult {
+interface EvaluationScore {
     readonly score: number;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly expectedAspectCount: number;
     readonly reasoning?: string;
     readonly rawAspects?: readonly string[];
-    readonly graderRawRequest?: JsonObject;
+    readonly evaluatorRawRequest?: JsonObject;
 }
-interface Grader {
+interface Evaluator {
     readonly kind: string;
-    grade(context: GradeContext): Promise<GradeResult> | GradeResult;
-}
-declare class HeuristicGrader implements Grader {
-    readonly kind = "heuristic";
-    grade(context: GradeContext): GradeResult;
+    evaluate(context: EvaluationContext): Promise<EvaluationScore> | EvaluationScore;
 }
-type JudgeProviderResolver = (context: GradeContext) => Promise<Provider | undefined>;
-interface QualityGraderOptions {
+type JudgeProviderResolver = (context: EvaluationContext) => Promise<Provider | undefined>;
+interface LlmJudgeEvaluatorOptions {
     readonly resolveJudgeProvider: JudgeProviderResolver;
     readonly maxOutputTokens?: number;
     readonly temperature?: number;
+    readonly customPrompt?: string;
 }
-declare class QualityGrader implements Grader {
+declare class LlmJudgeEvaluator implements Evaluator {
     readonly kind = "llm_judge";
     private readonly resolveJudgeProvider;
     private readonly maxOutputTokens?;
     private readonly temperature?;
-    constructor(options: QualityGraderOptions);
-    grade(context: GradeContext): Promise<GradeResult>;
+    private readonly customPrompt?;
+    constructor(options: LlmJudgeEvaluatorOptions);
+    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
+}
+interface CodeEvaluatorOptions {
+    readonly script: string;
+    readonly cwd?: string;
+    readonly agentTimeoutMs?: number;
+}
+declare class CodeEvaluator implements Evaluator {
+    readonly kind = "code";
+    private readonly script;
+    private readonly cwd?;
+    private readonly agentTimeoutMs?;
+    constructor(options: CodeEvaluatorOptions);
+    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
 }
 
 type MaybePromise<T> = T | Promise<T>;
@@ -386,7 +442,9 @@ interface RunEvalCaseOptions {
     readonly evalCase: EvalCase;
     readonly provider: Provider;
     readonly target: ResolvedTarget;
-    readonly graders: Partial<Record<string, Grader>>;
+    readonly evaluators: Partial<Record<string, Evaluator>> & {
+        readonly llm_judge: Evaluator;
+    };
     readonly now?: () => Date;
     readonly maxRetries?: number;
     readonly agentTimeoutMs?: number;
@@ -411,7 +469,7 @@ interface RunEvaluationOptions {
     readonly targets?: readonly TargetDefinition[];
     readonly env?: EnvLookup;
     readonly providerFactory?: (target: ResolvedTarget) => Provider;
-    readonly graders?: Partial<Record<string, Grader>>;
+    readonly evaluators?: Partial<Record<string, Evaluator>>;
     readonly maxRetries?: number;
     readonly agentTimeoutMs?: number;
     readonly promptDumpDir?: string;
@@ -432,4 +490,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationResult, GRADER_KINDS, type GeminiResolvedConfig, type GradeContext, type GradeResult, type Grader, type GraderKind, HeuristicGrader, type HeuristicScore, type JsonObject, type JsonPrimitive, type JsonValue, type MockResolvedConfig, type ProgressEvent, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, QualityGrader, type QualityGraderOptions, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, calculateHits, calculateMisses, createAgentKernel, createProvider, ensureVSCodeSubagents, extractAspects, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isErrorLike, isGraderKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, readTargetDefinitions, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, scoreCandidateResponse };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CodexResolvedConfig, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, readTargetDefinitions, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation };

package/dist/index.d.ts

@@ -75,19 +75,25 @@ declare function isJsonValue(value: unknown): value is JsonValue;
  * Guard validating raw test messages.
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const GRADER_KIND_VALUES: readonly ["heuristic", "llm_judge"];
-/**
- * Supported grader implementations.
- */
-declare const GRADER_KINDS: readonly ["heuristic", "llm_judge"];
-/**
- * Grader identifiers available to the pipeline.
- */
-type GraderKind = (typeof GRADER_KIND_VALUES)[number];
-/**
- * Guard validating grader identifiers.
- */
-declare function isGraderKind(value: unknown): value is GraderKind;
+declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge"];
+type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
+declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
+type CodeEvaluatorConfig = {
+    readonly name: string;
+    readonly type: "code";
+    readonly script: string;
+    readonly resolvedScriptPath?: string;
+    readonly cwd?: string;
+    readonly resolvedCwd?: string;
+};
+type LlmJudgeEvaluatorConfig = {
+    readonly name: string;
+    readonly type: "llm_judge";
+    readonly prompt?: string;
+    readonly promptPath?: string;
+    readonly model?: string;
+};
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
 /**
  * Test case definition sourced from AgentV specs.
  */
@@ -103,7 +109,8 @@ interface EvalCase {
     readonly file_paths: readonly string[];
     readonly code_snippets: readonly string[];
     readonly outcome: string;
-    readonly grader: GraderKind;
+    readonly evaluator?: EvaluatorKind;
+    readonly evaluators?: readonly EvaluatorConfig[];
 }
 /**
  * Evaluator scorecard for a single test case run.
@@ -121,7 +128,18 @@ interface EvaluationResult {
     readonly reasoning?: string;
     readonly raw_aspects?: readonly string[];
     readonly raw_request?: JsonObject;
-    readonly grader_raw_request?: JsonObject;
+    readonly evaluator_raw_request?: JsonObject;
+    readonly evaluator_results?: readonly EvaluatorResult[];
+}
+interface EvaluatorResult {
+    readonly name: string;
+    readonly type: EvaluatorKind;
+    readonly score: number;
+    readonly hits: readonly string[];
+    readonly misses: readonly string[];
+    readonly reasoning?: string;
+    readonly raw_request?: JsonObject;
+    readonly evaluator_raw_request?: JsonObject;
 }
 /**
  * Convenience accessor matching the Python hit_count property.
@@ -177,13 +195,13 @@ declare function resolveFileReference(rawValue: string, searchRoots: readonly st
 }>;
 
 type ChatPrompt = AxChatRequest["chatPrompt"];
-type ProviderKind = "azure" | "anthropic" | "gemini" | "mock" | "vscode" | "vscode-insiders";
+type ProviderKind = "azure" | "anthropic" | "gemini" | "codex" | "cli" | "mock" | "vscode" | "vscode-insiders";
 interface ProviderRequest {
     readonly prompt: string;
     readonly guidelines?: string;
     readonly guideline_patterns?: readonly string[];
     readonly chatPrompt?: ChatPrompt;
-    readonly attachments?: readonly string[];
+    readonly inputFiles?: readonly string[];
     readonly evalCaseId?: string;
     readonly attempt?: number;
     readonly maxOutputTokens?: number;
@@ -202,6 +220,15 @@ interface Provider {
     readonly kind: ProviderKind;
     readonly targetName: string;
     invoke(request: ProviderRequest): Promise<ProviderResponse>;
+    /**
+     * Optional capability marker for provider-managed batching (single session handling multiple requests).
+     */
+    readonly supportsBatch?: boolean;
+    /**
+     * Optional batch invocation hook. When defined alongside supportsBatch=true,
+     * the orchestrator may send multiple requests in a single provider session.
+     */
+    invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -233,6 +260,12 @@ interface GeminiResolvedConfig {
     readonly temperature?: number;
     readonly maxOutputTokens?: number;
 }
+interface CodexResolvedConfig {
+    readonly executable: string;
+    readonly args?: readonly string[];
+    readonly cwd?: string;
+    readonly timeoutMs?: number;
+}
 interface MockResolvedConfig {
     readonly response?: string;
     readonly delayMs?: number;
@@ -246,36 +279,73 @@ interface VSCodeResolvedConfig {
     readonly subagentRoot?: string;
     readonly workspaceTemplate?: string;
 }
+type CliHealthcheck = {
+    readonly type: "http";
+    readonly url: string;
+    readonly timeoutMs?: number;
+} | {
+    readonly type: "command";
+    readonly commandTemplate: string;
+    readonly timeoutMs?: number;
+    readonly cwd?: string;
+};
+interface CliResolvedConfig {
+    readonly commandTemplate: string;
+    readonly filesFormat?: string;
+    readonly cwd?: string;
+    readonly env?: Record<string, string>;
+    readonly timeoutMs?: number;
+    readonly healthcheck?: CliHealthcheck;
+}
 type ResolvedTarget = {
     readonly kind: "azure";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: AzureResolvedConfig;
 } | {
     readonly kind: "anthropic";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: AnthropicResolvedConfig;
 } | {
     readonly kind: "gemini";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: GeminiResolvedConfig;
+} | {
+    readonly kind: "codex";
+    readonly name: string;
+    readonly judgeTarget?: string;
+    readonly workers?: number;
+    readonly providerBatching?: boolean;
+    readonly config: CodexResolvedConfig;
 } | {
     readonly kind: "mock";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: MockResolvedConfig;
 } | {
     readonly kind: "vscode" | "vscode-insiders";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: VSCodeResolvedConfig;
+} | {
+    readonly kind: "cli";
+    readonly name: string;
+    readonly judgeTarget?: string;
+    readonly workers?: number;
+    readonly providerBatching?: boolean;
+    readonly config: CliResolvedConfig;
 };
 declare function resolveTargetDefinition(definition: TargetDefinition, env?: EnvLookup): ResolvedTarget;
 
@@ -303,36 +373,7 @@ declare function ensureVSCodeSubagents(options: EnsureSubagentsOptions): Promise
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
-interface HeuristicScore {
-    readonly score: number;
-    readonly hits: readonly string[];
-    readonly misses: readonly string[];
-    readonly hitCount: number;
-    readonly totalAspects: number;
-    readonly rawAspects: readonly string[];
-}
-/**
- * Extract individual evaluation aspects from the expected assistant response.
- */
-declare function extractAspects(expectedResponse: string): readonly string[];
-/**
- * Determine which aspects were covered by the candidate response.
- */
-declare function calculateHits(candidateResponse: string, expectedAspects: readonly string[]): readonly string[];
-/**
- * Determine which aspects were not satisfied by the candidate response.
- */
-declare function calculateMisses(candidateResponse: string, expectedAspects: readonly string[], resolvedHits?: readonly string[]): readonly string[];
-/**
- * Evaluate the candidate response against the expected aspects.
- */
-declare function scoreCandidateResponse(candidateResponse: string, expectedAspects: readonly string[]): HeuristicScore;
-/**
- * Detect common error-prefixed outputs from providers.
- */
-declare function isErrorLike(text: string | undefined | null): boolean;
-
-interface GradeContext {
+interface EvaluationContext {
     readonly evalCase: EvalCase;
     readonly candidate: string;
     readonly target: ResolvedTarget;
@@ -341,40 +382,55 @@ interface GradeContext {
     readonly promptInputs: {
         readonly request: string;
         readonly guidelines: string;
+        readonly systemMessage?: string;
     };
     readonly now: Date;
     readonly judgeProvider?: Provider;
+    readonly systemPrompt?: string;
+    readonly evaluator?: EvaluatorConfig;
+    readonly judgeModel?: string;
 }
-interface GradeResult {
+interface EvaluationScore {
     readonly score: number;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly expectedAspectCount: number;
     readonly reasoning?: string;
     readonly rawAspects?: readonly string[];
-    readonly graderRawRequest?: JsonObject;
+    readonly evaluatorRawRequest?: JsonObject;
 }
-interface Grader {
+interface Evaluator {
     readonly kind: string;
-    grade(context: GradeContext): Promise<GradeResult> | GradeResult;
-}
-declare class HeuristicGrader implements Grader {
-    readonly kind = "heuristic";
-    grade(context: GradeContext): GradeResult;
+    evaluate(context: EvaluationContext): Promise<EvaluationScore> | EvaluationScore;
 }
-type JudgeProviderResolver = (context: GradeContext) => Promise<Provider | undefined>;
-interface QualityGraderOptions {
+type JudgeProviderResolver = (context: EvaluationContext) => Promise<Provider | undefined>;
+interface LlmJudgeEvaluatorOptions {
     readonly resolveJudgeProvider: JudgeProviderResolver;
     readonly maxOutputTokens?: number;
     readonly temperature?: number;
+    readonly customPrompt?: string;
 }
-declare class QualityGrader implements Grader {
+declare class LlmJudgeEvaluator implements Evaluator {
     readonly kind = "llm_judge";
     private readonly resolveJudgeProvider;
     private readonly maxOutputTokens?;
     private readonly temperature?;
-    constructor(options: QualityGraderOptions);
-    grade(context: GradeContext): Promise<GradeResult>;
+    private readonly customPrompt?;
+    constructor(options: LlmJudgeEvaluatorOptions);
+    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
+}
+interface CodeEvaluatorOptions {
+    readonly script: string;
+    readonly cwd?: string;
+    readonly agentTimeoutMs?: number;
+}
+declare class CodeEvaluator implements Evaluator {
+    readonly kind = "code";
+    private readonly script;
+    private readonly cwd?;
+    private readonly agentTimeoutMs?;
+    constructor(options: CodeEvaluatorOptions);
+    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
 }
 
 type MaybePromise<T> = T | Promise<T>;
@@ -386,7 +442,9 @@ interface RunEvalCaseOptions {
     readonly evalCase: EvalCase;
     readonly provider: Provider;
     readonly target: ResolvedTarget;
-    readonly graders: Partial<Record<string, Grader>>;
+    readonly evaluators: Partial<Record<string, Evaluator>> & {
+        readonly llm_judge: Evaluator;
+    };
     readonly now?: () => Date;
     readonly maxRetries?: number;
     readonly agentTimeoutMs?: number;
@@ -411,7 +469,7 @@ interface RunEvaluationOptions {
     readonly targets?: readonly TargetDefinition[];
     readonly env?: EnvLookup;
     readonly providerFactory?: (target: ResolvedTarget) => Provider;
-    readonly graders?: Partial<Record<string, Grader>>;
+    readonly evaluators?: Partial<Record<string, Evaluator>>;
     readonly maxRetries?: number;
     readonly agentTimeoutMs?: number;
     readonly promptDumpDir?: string;
@@ -432,4 +490,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationResult, GRADER_KINDS, type GeminiResolvedConfig, type GradeContext, type GradeResult, type Grader, type GraderKind, HeuristicGrader, type HeuristicScore, type JsonObject, type JsonPrimitive, type JsonValue, type MockResolvedConfig, type ProgressEvent, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, QualityGrader, type QualityGraderOptions, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, calculateHits, calculateMisses, createAgentKernel, createProvider, ensureVSCodeSubagents, extractAspects, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isErrorLike, isGraderKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, readTargetDefinitions, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, scoreCandidateResponse };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CodexResolvedConfig, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, readTargetDefinitions, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation };