@agentv/core 0.22.1 → 0.23.0

package/dist/index.d.cts

@@ -75,7 +75,7 @@ declare function isJsonValue(value: unknown): value is JsonValue;
  * Guard validating raw test messages.
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge", "rubric"];
+declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite"];
 type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 type CodeEvaluatorConfig = {
@@ -91,6 +91,7 @@ type LlmJudgeEvaluatorConfig = {
     readonly type: 'llm_judge';
     readonly prompt?: string;
     readonly promptPath?: string;
+    readonly rubrics?: readonly RubricItem[];
 };
 type RubricItem = {
     readonly id: string;
@@ -98,12 +99,26 @@ type RubricItem = {
     readonly weight: number;
     readonly required: boolean;
 };
-type RubricEvaluatorConfig = {
+type CompositeAggregatorConfig = {
+    readonly type: 'weighted_average';
+    readonly weights?: Record<string, number>;
+} | {
+    readonly type: 'code_judge';
+    readonly path: string;
+    readonly cwd?: string;
+} | {
+    readonly type: 'llm_judge';
+    readonly prompt?: string;
+    readonly promptPath?: string;
+    readonly model?: string;
+};
+type CompositeEvaluatorConfig = {
     readonly name: string;
-    readonly type: 'rubric';
-    readonly rubrics: readonly RubricItem[];
+    readonly type: 'composite';
+    readonly evaluators: readonly EvaluatorConfig[];
+    readonly aggregator: CompositeAggregatorConfig;
 };
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | RubricEvaluatorConfig;
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig;
 /**
  * Eval case definition sourced from AgentV specs.
  */
@@ -150,12 +165,14 @@ interface EvaluatorResult {
     readonly name: string;
     readonly type: EvaluatorKind;
     readonly score: number;
+    readonly weight?: number;
     readonly verdict?: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly reasoning?: string;
     readonly raw_request?: JsonObject;
     readonly evaluator_provider_request?: JsonObject;
+    readonly evaluator_results?: readonly EvaluatorResult[];
 }
 /**
  * Convenience accessor matching the Python hit_count property.
@@ -530,20 +547,6 @@ declare function subscribeToCodexLogEntries(listener: CodexLogListener): () => v
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
-interface RubricEvaluatorOptions {
-    readonly config: RubricEvaluatorConfig;
-    readonly resolveJudgeProvider: (context: EvaluationContext) => Promise<Provider | undefined>;
-}
-declare class RubricEvaluator implements Evaluator {
-    readonly kind = "rubric";
-    private readonly config;
-    private readonly resolveJudgeProvider;
-    constructor(options: RubricEvaluatorOptions);
-    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
-    private buildPrompt;
-    private calculateScore;
-}
-
 interface EvaluationContext {
     readonly evalCase: EvalCase;
     readonly candidate: string;
@@ -563,13 +566,26 @@ interface EvaluationContext {
 }
 interface EvaluationScore {
     readonly score: number;
-    readonly verdict?: EvaluationVerdict;
+    readonly verdict: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly expectedAspectCount: number;
     readonly reasoning?: string;
     readonly rawAspects?: readonly string[];
     readonly evaluatorRawRequest?: JsonObject;
+    readonly evaluatorResults?: readonly ChildEvaluatorResult[];
+}
+interface ChildEvaluatorResult {
+    readonly name: string;
+    readonly type: string;
+    readonly score: number;
+    readonly weight?: number;
+    readonly verdict: EvaluationVerdict;
+    readonly hits: readonly string[];
+    readonly misses: readonly string[];
+    readonly reasoning?: string;
+    readonly evaluatorRawRequest?: JsonObject;
+    readonly evaluatorResults?: readonly ChildEvaluatorResult[];
 }
 interface Evaluator {
     readonly kind: string;
@@ -590,7 +606,10 @@ declare class LlmJudgeEvaluator implements Evaluator {
     private readonly evaluatorTemplate?;
     constructor(options: LlmJudgeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
-    private evaluateWithPrompt;
+    private evaluateFreeform;
+    private evaluateWithRubrics;
+    private buildRubricPrompt;
+    private runWithRetry;
 }
 interface CodeEvaluatorOptions {
     readonly script: string;
@@ -605,6 +624,26 @@ declare class CodeEvaluator implements Evaluator {
     constructor(options: CodeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
 }
+interface EvaluatorFactory {
+    create(config: EvaluatorConfig, context: EvaluationContext): Evaluator;
+}
+interface CompositeEvaluatorOptions {
+    readonly config: CompositeEvaluatorConfig;
+    readonly evaluatorFactory: EvaluatorFactory;
+    readonly cwd?: string;
+}
+declare class CompositeEvaluator implements Evaluator {
+    readonly kind = "composite";
+    private readonly config;
+    private readonly evaluatorFactory;
+    private readonly cwd?;
+    constructor(options: CompositeEvaluatorOptions);
+    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
+    private aggregate;
+    private runWeightedAverage;
+    private runCodeAggregator;
+    private runLlmAggregator;
+}
 
 type MaybePromise<T> = T | Promise<T>;
 interface EvaluationCache {
@@ -675,4 +714,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, RubricEvaluator, type RubricEvaluatorConfig, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.d.ts

@@ -75,7 +75,7 @@ declare function isJsonValue(value: unknown): value is JsonValue;
  * Guard validating raw test messages.
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge", "rubric"];
+declare const EVALUATOR_KIND_VALUES: readonly ["code_judge", "llm_judge", "rubric", "composite"];
 type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 type CodeEvaluatorConfig = {
@@ -91,6 +91,7 @@ type LlmJudgeEvaluatorConfig = {
     readonly type: 'llm_judge';
     readonly prompt?: string;
     readonly promptPath?: string;
+    readonly rubrics?: readonly RubricItem[];
 };
 type RubricItem = {
     readonly id: string;
@@ -98,12 +99,26 @@ type RubricItem = {
     readonly weight: number;
     readonly required: boolean;
 };
-type RubricEvaluatorConfig = {
+type CompositeAggregatorConfig = {
+    readonly type: 'weighted_average';
+    readonly weights?: Record<string, number>;
+} | {
+    readonly type: 'code_judge';
+    readonly path: string;
+    readonly cwd?: string;
+} | {
+    readonly type: 'llm_judge';
+    readonly prompt?: string;
+    readonly promptPath?: string;
+    readonly model?: string;
+};
+type CompositeEvaluatorConfig = {
     readonly name: string;
-    readonly type: 'rubric';
-    readonly rubrics: readonly RubricItem[];
+    readonly type: 'composite';
+    readonly evaluators: readonly EvaluatorConfig[];
+    readonly aggregator: CompositeAggregatorConfig;
 };
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | RubricEvaluatorConfig;
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | CompositeEvaluatorConfig;
 /**
  * Eval case definition sourced from AgentV specs.
  */
@@ -150,12 +165,14 @@ interface EvaluatorResult {
     readonly name: string;
     readonly type: EvaluatorKind;
     readonly score: number;
+    readonly weight?: number;
     readonly verdict?: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly reasoning?: string;
     readonly raw_request?: JsonObject;
     readonly evaluator_provider_request?: JsonObject;
+    readonly evaluator_results?: readonly EvaluatorResult[];
 }
 /**
  * Convenience accessor matching the Python hit_count property.
@@ -530,20 +547,6 @@ declare function subscribeToCodexLogEntries(listener: CodexLogListener): () => v
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
-interface RubricEvaluatorOptions {
-    readonly config: RubricEvaluatorConfig;
-    readonly resolveJudgeProvider: (context: EvaluationContext) => Promise<Provider | undefined>;
-}
-declare class RubricEvaluator implements Evaluator {
-    readonly kind = "rubric";
-    private readonly config;
-    private readonly resolveJudgeProvider;
-    constructor(options: RubricEvaluatorOptions);
-    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
-    private buildPrompt;
-    private calculateScore;
-}
-
 interface EvaluationContext {
     readonly evalCase: EvalCase;
     readonly candidate: string;
@@ -563,13 +566,26 @@ interface EvaluationContext {
 }
 interface EvaluationScore {
     readonly score: number;
-    readonly verdict?: EvaluationVerdict;
+    readonly verdict: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly expectedAspectCount: number;
     readonly reasoning?: string;
     readonly rawAspects?: readonly string[];
     readonly evaluatorRawRequest?: JsonObject;
+    readonly evaluatorResults?: readonly ChildEvaluatorResult[];
+}
+interface ChildEvaluatorResult {
+    readonly name: string;
+    readonly type: string;
+    readonly score: number;
+    readonly weight?: number;
+    readonly verdict: EvaluationVerdict;
+    readonly hits: readonly string[];
+    readonly misses: readonly string[];
+    readonly reasoning?: string;
+    readonly evaluatorRawRequest?: JsonObject;
+    readonly evaluatorResults?: readonly ChildEvaluatorResult[];
 }
 interface Evaluator {
     readonly kind: string;
@@ -590,7 +606,10 @@ declare class LlmJudgeEvaluator implements Evaluator {
     private readonly evaluatorTemplate?;
     constructor(options: LlmJudgeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
-    private evaluateWithPrompt;
+    private evaluateFreeform;
+    private evaluateWithRubrics;
+    private buildRubricPrompt;
+    private runWithRetry;
 }
 interface CodeEvaluatorOptions {
     readonly script: string;
@@ -605,6 +624,26 @@ declare class CodeEvaluator implements Evaluator {
     constructor(options: CodeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
 }
+interface EvaluatorFactory {
+    create(config: EvaluatorConfig, context: EvaluationContext): Evaluator;
+}
+interface CompositeEvaluatorOptions {
+    readonly config: CompositeEvaluatorConfig;
+    readonly evaluatorFactory: EvaluatorFactory;
+    readonly cwd?: string;
+}
+declare class CompositeEvaluator implements Evaluator {
+    readonly kind = "composite";
+    private readonly config;
+    private readonly evaluatorFactory;
+    private readonly cwd?;
+    constructor(options: CompositeEvaluatorOptions);
+    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
+    private aggregate;
+    private runWeightedAverage;
+    private runCodeAggregator;
+    private runLlmAggregator;
+}
 
 type MaybePromise<T> = T | Promise<T>;
 interface EvaluationCache {
@@ -675,4 +714,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, RubricEvaluator, type RubricEvaluatorConfig, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type ChildEvaluatorResult, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type CompositeAggregatorConfig, CompositeEvaluator, type CompositeEvaluatorConfig, type CompositeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorFactory, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };