@agentv/core 0.20.0 → 0.22.0

package/dist/index.d.cts

@@ -1,3 +1,5 @@
+import * as ai from 'ai';
+
 /**
  * JSON primitive values appearing in AgentV payloads.
  */
@@ -73,7 +75,7 @@ declare function isJsonValue(value: unknown): value is JsonValue;
  * Guard validating raw test messages.
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge"];
+declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge", "rubric"];
 type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 type CodeEvaluatorConfig = {
@@ -90,7 +92,18 @@ type LlmJudgeEvaluatorConfig = {
     readonly prompt?: string;
     readonly promptPath?: string;
 };
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
+type RubricItem = {
+    readonly id: string;
+    readonly description: string;
+    readonly weight: number;
+    readonly required: boolean;
+};
+type RubricEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'rubric';
+    readonly rubrics: readonly RubricItem[];
+};
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | RubricEvaluatorConfig;
 /**
  * Eval case definition sourced from AgentV specs.
  */
@@ -132,10 +145,12 @@ interface EvaluationResult {
     readonly evaluator_results?: readonly EvaluatorResult[];
     readonly error?: string;
 }
+type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
 interface EvaluatorResult {
     readonly name: string;
     readonly type: EvaluatorKind;
     readonly score: number;
+    readonly verdict?: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly reasoning?: string;
@@ -189,6 +204,11 @@ interface Provider {
      * the orchestrator may send multiple requests in a single provider session.
      */
     invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
+    /**
+     * Optional method to get a Vercel AI SDK LanguageModel instance for structured output generation.
+     * Used by evaluators that need generateObject/generateText from the AI SDK.
+     */
+    asLanguageModel?(): ai.LanguageModel;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -510,6 +530,20 @@ declare function subscribeToCodexLogEntries(listener: CodexLogListener): () => v
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
+interface RubricEvaluatorOptions {
+    readonly config: RubricEvaluatorConfig;
+    readonly resolveJudgeProvider: (context: EvaluationContext) => Promise<Provider | undefined>;
+}
+declare class RubricEvaluator implements Evaluator {
+    readonly kind = "rubric";
+    private readonly config;
+    private readonly resolveJudgeProvider;
+    constructor(options: RubricEvaluatorOptions);
+    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
+    private buildPrompt;
+    private calculateScore;
+}
+
 interface EvaluationContext {
     readonly evalCase: EvalCase;
     readonly candidate: string;
@@ -529,6 +563,7 @@ interface EvaluationContext {
 }
 interface EvaluationScore {
     readonly score: number;
+    readonly verdict?: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly expectedAspectCount: number;
@@ -624,9 +659,20 @@ interface RunEvaluationOptions {
 declare function runEvaluation(options: RunEvaluationOptions): Promise<readonly EvaluationResult[]>;
 declare function runEvalCase(options: RunEvalCaseOptions): Promise<EvaluationResult>;
 
+interface GenerateRubricsOptions {
+    readonly expectedOutcome: string;
+    readonly question?: string;
+    readonly referenceAnswer?: string;
+    readonly provider: Provider;
+}
+/**
+ * Generate rubrics from expected outcome using an LLM.
+ */
+declare function generateRubrics(options: GenerateRubricsOptions): Promise<readonly RubricItem[]>;
+
 type AgentKernel = {
     status: string;
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, RubricEvaluator, type RubricEvaluatorConfig, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.d.ts

@@ -1,3 +1,5 @@
+import * as ai from 'ai';
+
 /**
  * JSON primitive values appearing in AgentV payloads.
  */
@@ -73,7 +75,7 @@ declare function isJsonValue(value: unknown): value is JsonValue;
  * Guard validating raw test messages.
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge"];
+declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge", "rubric"];
 type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 type CodeEvaluatorConfig = {
@@ -90,7 +92,18 @@ type LlmJudgeEvaluatorConfig = {
     readonly prompt?: string;
     readonly promptPath?: string;
 };
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
+type RubricItem = {
+    readonly id: string;
+    readonly description: string;
+    readonly weight: number;
+    readonly required: boolean;
+};
+type RubricEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'rubric';
+    readonly rubrics: readonly RubricItem[];
+};
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | RubricEvaluatorConfig;
 /**
  * Eval case definition sourced from AgentV specs.
  */
@@ -132,10 +145,12 @@ interface EvaluationResult {
     readonly evaluator_results?: readonly EvaluatorResult[];
     readonly error?: string;
 }
+type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
 interface EvaluatorResult {
     readonly name: string;
     readonly type: EvaluatorKind;
     readonly score: number;
+    readonly verdict?: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly reasoning?: string;
@@ -189,6 +204,11 @@ interface Provider {
      * the orchestrator may send multiple requests in a single provider session.
      */
     invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
+    /**
+     * Optional method to get a Vercel AI SDK LanguageModel instance for structured output generation.
+     * Used by evaluators that need generateObject/generateText from the AI SDK.
+     */
+    asLanguageModel?(): ai.LanguageModel;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -510,6 +530,20 @@ declare function subscribeToCodexLogEntries(listener: CodexLogListener): () => v
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
+interface RubricEvaluatorOptions {
+    readonly config: RubricEvaluatorConfig;
+    readonly resolveJudgeProvider: (context: EvaluationContext) => Promise<Provider | undefined>;
+}
+declare class RubricEvaluator implements Evaluator {
+    readonly kind = "rubric";
+    private readonly config;
+    private readonly resolveJudgeProvider;
+    constructor(options: RubricEvaluatorOptions);
+    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
+    private buildPrompt;
+    private calculateScore;
+}
+
 interface EvaluationContext {
     readonly evalCase: EvalCase;
     readonly candidate: string;
@@ -529,6 +563,7 @@ interface EvaluationContext {
 }
 interface EvaluationScore {
     readonly score: number;
+    readonly verdict?: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly expectedAspectCount: number;
@@ -624,9 +659,20 @@ interface RunEvaluationOptions {
 declare function runEvaluation(options: RunEvaluationOptions): Promise<readonly EvaluationResult[]>;
 declare function runEvalCase(options: RunEvalCaseOptions): Promise<EvaluationResult>;
 
+interface GenerateRubricsOptions {
+    readonly expectedOutcome: string;
+    readonly question?: string;
+    readonly referenceAnswer?: string;
+    readonly provider: Provider;
+}
+/**
+ * Generate rubrics from expected outcome using an LLM.
+ */
+declare function generateRubrics(options: GenerateRubricsOptions): Promise<readonly RubricItem[]>;
+
 type AgentKernel = {
     status: string;
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, RubricEvaluator, type RubricEvaluatorConfig, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };