@agentv/core 0.22.1 → 0.22.2

package/dist/index.d.cts

@@ -91,6 +91,7 @@ type LlmJudgeEvaluatorConfig = {
     readonly type: 'llm_judge';
     readonly prompt?: string;
     readonly promptPath?: string;
+    readonly rubrics?: readonly RubricItem[];
 };
 type RubricItem = {
     readonly id: string;
@@ -98,12 +99,7 @@ type RubricItem = {
     readonly weight: number;
     readonly required: boolean;
 };
-type RubricEvaluatorConfig = {
-    readonly name: string;
-    readonly type: 'rubric';
-    readonly rubrics: readonly RubricItem[];
-};
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | RubricEvaluatorConfig;
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
 /**
  * Eval case definition sourced from AgentV specs.
  */
@@ -530,20 +526,6 @@ declare function subscribeToCodexLogEntries(listener: CodexLogListener): () => v
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
-interface RubricEvaluatorOptions {
-    readonly config: RubricEvaluatorConfig;
-    readonly resolveJudgeProvider: (context: EvaluationContext) => Promise<Provider | undefined>;
-}
-declare class RubricEvaluator implements Evaluator {
-    readonly kind = "rubric";
-    private readonly config;
-    private readonly resolveJudgeProvider;
-    constructor(options: RubricEvaluatorOptions);
-    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
-    private buildPrompt;
-    private calculateScore;
-}
-
 interface EvaluationContext {
     readonly evalCase: EvalCase;
     readonly candidate: string;
@@ -563,7 +545,7 @@ interface EvaluationContext {
 }
 interface EvaluationScore {
     readonly score: number;
-    readonly verdict?: EvaluationVerdict;
+    readonly verdict: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly expectedAspectCount: number;
@@ -590,7 +572,10 @@ declare class LlmJudgeEvaluator implements Evaluator {
     private readonly evaluatorTemplate?;
     constructor(options: LlmJudgeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
-    private evaluateWithPrompt;
+    private evaluateFreeform;
+    private evaluateWithRubrics;
+    private buildRubricPrompt;
+    private runWithRetry;
 }
 interface CodeEvaluatorOptions {
     readonly script: string;
@@ -675,4 +660,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, RubricEvaluator, type RubricEvaluatorConfig, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.d.ts

@@ -91,6 +91,7 @@ type LlmJudgeEvaluatorConfig = {
     readonly type: 'llm_judge';
     readonly prompt?: string;
     readonly promptPath?: string;
+    readonly rubrics?: readonly RubricItem[];
 };
 type RubricItem = {
     readonly id: string;
@@ -98,12 +99,7 @@ type RubricItem = {
     readonly weight: number;
     readonly required: boolean;
 };
-type RubricEvaluatorConfig = {
-    readonly name: string;
-    readonly type: 'rubric';
-    readonly rubrics: readonly RubricItem[];
-};
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | RubricEvaluatorConfig;
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
 /**
  * Eval case definition sourced from AgentV specs.
  */
@@ -530,20 +526,6 @@ declare function subscribeToCodexLogEntries(listener: CodexLogListener): () => v
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
-interface RubricEvaluatorOptions {
-    readonly config: RubricEvaluatorConfig;
-    readonly resolveJudgeProvider: (context: EvaluationContext) => Promise<Provider | undefined>;
-}
-declare class RubricEvaluator implements Evaluator {
-    readonly kind = "rubric";
-    private readonly config;
-    private readonly resolveJudgeProvider;
-    constructor(options: RubricEvaluatorOptions);
-    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
-    private buildPrompt;
-    private calculateScore;
-}
-
 interface EvaluationContext {
     readonly evalCase: EvalCase;
     readonly candidate: string;
@@ -563,7 +545,7 @@ interface EvaluationContext {
 }
 interface EvaluationScore {
     readonly score: number;
-    readonly verdict?: EvaluationVerdict;
+    readonly verdict: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly expectedAspectCount: number;
@@ -590,7 +572,10 @@ declare class LlmJudgeEvaluator implements Evaluator {
     private readonly evaluatorTemplate?;
     constructor(options: LlmJudgeEvaluatorOptions);
     evaluate(context: EvaluationContext): Promise<EvaluationScore>;
-    private evaluateWithPrompt;
+    private evaluateFreeform;
+    private evaluateWithRubrics;
+    private buildRubricPrompt;
+    private runWithRetry;
 }
 interface CodeEvaluatorOptions {
     readonly script: string;
@@ -675,4 +660,4 @@ type AgentKernel = {
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, RubricEvaluator, type RubricEvaluatorConfig, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };