@agentv/core 0.21.0 → 0.22.0

package/dist/index.d.cts

@@ -1,3 +1,5 @@
+import * as ai from 'ai';
+
 /**
  * JSON primitive values appearing in AgentV payloads.
  */
@@ -73,7 +75,7 @@ declare function isJsonValue(value: unknown): value is JsonValue;
  * Guard validating raw test messages.
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge"];
+declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge", "rubric"];
 type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 type CodeEvaluatorConfig = {
@@ -90,7 +92,18 @@ type LlmJudgeEvaluatorConfig = {
     readonly prompt?: string;
     readonly promptPath?: string;
 };
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
+type RubricItem = {
+    readonly id: string;
+    readonly description: string;
+    readonly weight: number;
+    readonly required: boolean;
+};
+type RubricEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'rubric';
+    readonly rubrics: readonly RubricItem[];
+};
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | RubricEvaluatorConfig;
 /**
  * Eval case definition sourced from AgentV specs.
  */
@@ -132,10 +145,12 @@ interface EvaluationResult {
     readonly evaluator_results?: readonly EvaluatorResult[];
     readonly error?: string;
 }
+type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
 interface EvaluatorResult {
     readonly name: string;
     readonly type: EvaluatorKind;
     readonly score: number;
+    readonly verdict?: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly reasoning?: string;
@@ -189,6 +204,11 @@ interface Provider {
      * the orchestrator may send multiple requests in a single provider session.
      */
     invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
+    /**
+     * Optional method to get a Vercel AI SDK LanguageModel instance for structured output generation.
+     * Used by evaluators that need generateObject/generateText from the AI SDK.
+     */
+    asLanguageModel?(): ai.LanguageModel;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -510,6 +530,20 @@ declare function subscribeToCodexLogEntries(listener: CodexLogListener): () => v
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
+interface RubricEvaluatorOptions {
+    readonly config: RubricEvaluatorConfig;
+    readonly resolveJudgeProvider: (context: EvaluationContext) => Promise<Provider | undefined>;
+}
+declare class RubricEvaluator implements Evaluator {
+    readonly kind = "rubric";
+    private readonly config;
+    private readonly resolveJudgeProvider;
+    constructor(options: RubricEvaluatorOptions);
+    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
+    private buildPrompt;
+    private calculateScore;
+}
+
 interface EvaluationContext {
     readonly evalCase: EvalCase;
     readonly candidate: string;
@@ -529,6 +563,7 @@ interface EvaluationContext {
 }
 interface EvaluationScore {
     readonly score: number;
+    readonly verdict?: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly expectedAspectCount: number;
@@ -624,9 +659,20 @@ interface RunEvaluationOptions {
 declare function runEvaluation(options: RunEvaluationOptions): Promise<readonly EvaluationResult[]>;
 declare function runEvalCase(options: RunEvalCaseOptions): Promise<EvaluationResult>;
 
+interface GenerateRubricsOptions {
+    readonly expectedOutcome: string;
+    readonly question?: string;
+    readonly referenceAnswer?: string;
+    readonly provider: Provider;
+}
+/**
+ * Generate rubrics from expected outcome using an LLM.
+ */
+declare function generateRubrics(options: GenerateRubricsOptions): Promise<readonly RubricItem[]>;
+
 type AgentKernel = {
     status: string;
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, RubricEvaluator, type RubricEvaluatorConfig, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.d.ts

@@ -1,3 +1,5 @@
+import * as ai from 'ai';
+
 /**
  * JSON primitive values appearing in AgentV payloads.
  */
@@ -73,7 +75,7 @@ declare function isJsonValue(value: unknown): value is JsonValue;
  * Guard validating raw test messages.
  */
 declare function isTestMessage(value: unknown): value is TestMessage;
-declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge"];
+declare const EVALUATOR_KIND_VALUES: readonly ["code", "llm_judge", "rubric"];
 type EvaluatorKind = (typeof EVALUATOR_KIND_VALUES)[number];
 declare function isEvaluatorKind(value: unknown): value is EvaluatorKind;
 type CodeEvaluatorConfig = {
@@ -90,7 +92,18 @@ type LlmJudgeEvaluatorConfig = {
     readonly prompt?: string;
     readonly promptPath?: string;
 };
-type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig;
+type RubricItem = {
+    readonly id: string;
+    readonly description: string;
+    readonly weight: number;
+    readonly required: boolean;
+};
+type RubricEvaluatorConfig = {
+    readonly name: string;
+    readonly type: 'rubric';
+    readonly rubrics: readonly RubricItem[];
+};
+type EvaluatorConfig = CodeEvaluatorConfig | LlmJudgeEvaluatorConfig | RubricEvaluatorConfig;
 /**
  * Eval case definition sourced from AgentV specs.
  */
@@ -132,10 +145,12 @@ interface EvaluationResult {
     readonly evaluator_results?: readonly EvaluatorResult[];
     readonly error?: string;
 }
+type EvaluationVerdict = 'pass' | 'fail' | 'borderline';
 interface EvaluatorResult {
     readonly name: string;
     readonly type: EvaluatorKind;
     readonly score: number;
+    readonly verdict?: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly reasoning?: string;
@@ -189,6 +204,11 @@ interface Provider {
      * the orchestrator may send multiple requests in a single provider session.
      */
     invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
+    /**
+     * Optional method to get a Vercel AI SDK LanguageModel instance for structured output generation.
+     * Used by evaluators that need generateObject/generateText from the AI SDK.
+     */
+    asLanguageModel?(): ai.LanguageModel;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -510,6 +530,20 @@ declare function subscribeToCodexLogEntries(listener: CodexLogListener): () => v
 declare function createProvider(target: ResolvedTarget): Provider;
 declare function resolveAndCreateProvider(definition: TargetDefinition, env?: EnvLookup): Provider;
 
+interface RubricEvaluatorOptions {
+    readonly config: RubricEvaluatorConfig;
+    readonly resolveJudgeProvider: (context: EvaluationContext) => Promise<Provider | undefined>;
+}
+declare class RubricEvaluator implements Evaluator {
+    readonly kind = "rubric";
+    private readonly config;
+    private readonly resolveJudgeProvider;
+    constructor(options: RubricEvaluatorOptions);
+    evaluate(context: EvaluationContext): Promise<EvaluationScore>;
+    private buildPrompt;
+    private calculateScore;
+}
+
 interface EvaluationContext {
     readonly evalCase: EvalCase;
     readonly candidate: string;
@@ -529,6 +563,7 @@ interface EvaluationContext {
 }
 interface EvaluationScore {
     readonly score: number;
+    readonly verdict?: EvaluationVerdict;
     readonly hits: readonly string[];
     readonly misses: readonly string[];
     readonly expectedAspectCount: number;
@@ -624,9 +659,20 @@ interface RunEvaluationOptions {
 declare function runEvaluation(options: RunEvaluationOptions): Promise<readonly EvaluationResult[]>;
 declare function runEvalCase(options: RunEvalCaseOptions): Promise<EvaluationResult>;
 
+interface GenerateRubricsOptions {
+    readonly expectedOutcome: string;
+    readonly question?: string;
+    readonly referenceAnswer?: string;
+    readonly provider: Provider;
+}
+/**
+ * Generate rubrics from expected outcome using an LLM.
+ */
+declare function generateRubrics(options: GenerateRubricsOptions): Promise<readonly RubricItem[]>;
+
 type AgentKernel = {
     status: string;
 };
 declare function createAgentKernel(): AgentKernel;
 
-export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };
+export { type AgentKernel, type AnthropicResolvedConfig, type AssistantTestMessage, type AzureResolvedConfig, type CliResolvedConfig, CodeEvaluator, type CodeEvaluatorConfig, type CodeEvaluatorOptions, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EvalCase, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type Evaluator, type EvaluatorConfig, type EvaluatorKind, type EvaluatorResult, type GeminiResolvedConfig, type GenerateRubricsOptions, type JsonObject, type JsonPrimitive, type JsonValue, LlmJudgeEvaluator, type LlmJudgeEvaluatorConfig, type LlmJudgeEvaluatorOptions, type MockResolvedConfig, type ProgressEvent, type PromptInputs, type Provider, type ProviderKind, type ProviderRequest, type ProviderResponse, type ResolvedTarget, RubricEvaluator, type RubricEvaluatorConfig, type RubricItem, type RunEvalCaseOptions, type RunEvaluationOptions, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetDefinition, type TestMessage, type TestMessageContent, type TestMessageRole, type ToolTestMessage, type UserTestMessage, type VSCodeResolvedConfig, buildDirectoryChain, buildPromptInputs, buildSearchRoots, consumeCodexLogEntries, createAgentKernel, createProvider, ensureVSCodeSubagents, extractCodeBlocks, fileExists, findGitRoot, generateRubrics, getHitCount, isEvaluatorKind, isGuidelineFile, isJsonObject, isJsonValue, isTestMessage, isTestMessageRole, listTargetNames, loadEvalCases, normalizeLineEndings, readTargetDefinitions, readTestSuiteMetadata, readTextFile, resolveAndCreateProvider, resolveFileReference, resolveTargetDefinition, runEvalCase, runEvaluation, subscribeToCodexLogEntries };

package/dist/index.js

@@ -8,7 +8,7 @@ import {
   readTextFile,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-SVY324GN.js";
+} from "./chunk-BO7KG7JX.js";
 
 // src/evaluation/types.ts
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
@@ -51,7 +51,7 @@ function isTestMessage(value) {
   }
   return candidate.content.every(isJsonObject);
 }
-var EVALUATOR_KIND_VALUES = ["code", "llm_judge"];
+var EVALUATOR_KIND_VALUES = ["code", "llm_judge", "rubric"];
 var EVALUATOR_KIND_SET = new Set(EVALUATOR_KIND_VALUES);
 function isEvaluatorKind(value) {
   return typeof value === "string" && EVALUATOR_KIND_SET.has(value);
@@ -453,6 +453,29 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
       }
     }
     const _model = asString2(rawEvaluator.model);
+    if (typeValue === "rubric") {
+      const rubrics = rawEvaluator.rubrics;
+      if (!Array.isArray(rubrics)) {
+        logWarning2(`Skipping rubric evaluator '${name}' in '${evalId}': missing rubrics array`);
+        continue;
+      }
+      const parsedRubrics = rubrics.filter((r) => isJsonObject2(r)).map((rubric, index) => ({
+        id: asString2(rubric.id) ?? `rubric-${index + 1}`,
+        description: asString2(rubric.description) ?? "",
+        weight: typeof rubric.weight === "number" ? rubric.weight : 1,
+        required: typeof rubric.required === "boolean" ? rubric.required : true
+      })).filter((r) => r.description.length > 0);
+      if (parsedRubrics.length === 0) {
+        logWarning2(`Skipping rubric evaluator '${name}' in '${evalId}': no valid rubrics found`);
+        continue;
+      }
+      evaluators.push({
+        name,
+        type: "rubric",
+        rubrics: parsedRubrics
+      });
+      continue;
+    }
     evaluators.push({
       name,
       type: "llm_judge",
@@ -933,7 +956,7 @@ async function loadEvalCases(evalFilePath, repoRoot, options) {
       continue;
     }
     const conversationId = asString5(evalcase.conversation_id);
-    const outcome = asString5(evalcase.outcome);
+    const outcome = asString5(evalcase.expected_outcome) ?? asString5(evalcase.outcome);
     const inputMessagesValue = evalcase.input_messages;
     const expectedMessagesValue = evalcase.expected_messages;
     if (!id || !outcome || !Array.isArray(inputMessagesValue)) {
@@ -987,6 +1010,33 @@ async function loadEvalCases(evalFilePath, repoRoot, options) {
       logError(`Skipping eval case '${id}': ${message}`);
       continue;
     }
+    const inlineRubrics = evalcase.rubrics;
+    if (inlineRubrics !== void 0 && Array.isArray(inlineRubrics)) {
+      const rubricItems = inlineRubrics.filter((r) => isJsonObject(r) || typeof r === "string").map((rubric, index) => {
+        if (typeof rubric === "string") {
+          return {
+            id: `rubric-${index + 1}`,
+            description: rubric,
+            weight: 1,
+            required: true
+          };
+        }
+        return {
+          id: asString5(rubric.id) ?? `rubric-${index + 1}`,
+          description: asString5(rubric.description) ?? "",
+          weight: typeof rubric.weight === "number" ? rubric.weight : 1,
+          required: typeof rubric.required === "boolean" ? rubric.required : true
+        };
+      }).filter((r) => r.description.length > 0);
+      if (rubricItems.length > 0) {
+        const rubricEvaluator = {
+          name: "rubric",
+          type: "rubric",
+          rubrics: rubricItems
+        };
+        evaluators = evaluators ? [rubricEvaluator, ...evaluators] : [rubricEvaluator];
+      }
+    }
     const userFilePaths = [];
     for (const segment of inputSegments) {
       if (segment.type === "file" && typeof segment.resolvedPath === "string") {
@@ -1085,6 +1135,9 @@ var AzureProvider = class {
       retryConfig: this.retryConfig
     });
   }
+  asLanguageModel() {
+    return this.model;
+  }
 };
 var AnthropicProvider = class {
   constructor(targetName, config) {
@@ -1118,6 +1171,9 @@ var AnthropicProvider = class {
       providerOptions
     });
   }
+  asLanguageModel() {
+    return this.model;
+  }
 };
 var GeminiProvider = class {
   constructor(targetName, config) {
@@ -1148,6 +1204,9 @@ var GeminiProvider = class {
       retryConfig: this.retryConfig
     });
   }
+  asLanguageModel() {
+    return this.model;
+  }
 };
 function buildAzureOptions(config) {
   const options = {
@@ -2869,6 +2928,148 @@ function resolveAndCreateProvider(definition, env = process.env) {
   return createProvider(resolved);
 }
 
+// src/evaluation/evaluators/rubric-evaluator.ts
+import { generateText as generateText2 } from "ai";
+import { z } from "zod";
+var rubricCheckResultSchema = z.object({
+  id: z.string().describe("The ID of the rubric item being checked"),
+  satisfied: z.boolean().describe("Whether this rubric requirement is met"),
+  reasoning: z.string().describe("Brief explanation (1-2 sentences) for this check")
+});
+var rubricEvaluationSchema = z.object({
+  checks: z.array(rubricCheckResultSchema).describe("Results for each rubric item"),
+  overall_reasoning: z.string().describe("Overall assessment summary (1-2 sentences)")
+});
+var RubricEvaluator = class {
+  kind = "rubric";
+  config;
+  resolveJudgeProvider;
+  constructor(options) {
+    this.config = options.config;
+    this.resolveJudgeProvider = options.resolveJudgeProvider;
+  }
+  async evaluate(context) {
+    const judgeProvider = await this.resolveJudgeProvider(context);
+    if (!judgeProvider) {
+      throw new Error("No judge provider available for rubric evaluation");
+    }
+    if (!this.config.rubrics || this.config.rubrics.length === 0) {
+      throw new Error(
+        `No rubrics found for evaluator "${this.config.name}". Run "agentv generate rubrics" first.`
+      );
+    }
+    const prompt = this.buildPrompt(context, this.config.rubrics);
+    const model = judgeProvider.asLanguageModel?.();
+    if (!model) {
+      throw new Error("Judge provider does not support language model interface");
+    }
+    const system = `You are an expert evaluator. Evaluate the candidate answer against each rubric item.
+You must return a valid JSON object matching this schema:
+{
+  "checks": [
+    {
+      "id": "string (rubric id)",
+      "satisfied": boolean,
+      "reasoning": "string (brief explanation)"
+    }
+  ],
+  "overall_reasoning": "string (summary)"
+}`;
+    let result;
+    let lastError;
+    for (let attempt = 1; attempt <= 3; attempt++) {
+      try {
+        const { text } = await generateText2({
+          model,
+          system,
+          prompt
+        });
+        const cleaned = text.replace(/```json\n?|```/g, "").trim();
+        result = rubricEvaluationSchema.parse(JSON.parse(cleaned));
+        break;
+      } catch (e) {
+        lastError = e instanceof Error ? e : new Error(String(e));
+      }
+    }
+    if (!result) {
+      throw new Error(
+        `Failed to parse rubric evaluation result after 3 attempts: ${lastError?.message}`
+      );
+    }
+    const { score, verdict, hits, misses } = this.calculateScore(result, this.config.rubrics);
+    return {
+      score,
+      verdict,
+      hits,
+      misses,
+      expectedAspectCount: this.config.rubrics.length,
+      reasoning: result.overall_reasoning,
+      evaluatorRawRequest: {
+        prompt
+      }
+    };
+  }
+  buildPrompt(context, rubrics) {
+    const parts = [
+      "You are an expert evaluator. Evaluate the candidate answer against each rubric item below.",
+      "",
+      "[[ ## question ## ]]",
+      context.evalCase.question,
+      "",
+      "[[ ## expected_outcome ## ]]",
+      context.evalCase.expected_outcome,
+      ""
+    ];
+    if (context.evalCase.reference_answer && context.evalCase.reference_answer.trim().length > 0) {
+      parts.push("[[ ## reference_answer ## ]]", context.evalCase.reference_answer, "");
+    }
+    parts.push("[[ ## candidate_answer ## ]]", context.candidate, "", "[[ ## rubrics ## ]]");
+    for (const rubric of rubrics) {
+      const requiredLabel = rubric.required ? " (REQUIRED)" : "";
+      const weightLabel = rubric.weight !== 1 ? ` (weight: ${rubric.weight})` : "";
+      parts.push(`- [${rubric.id}]${requiredLabel}${weightLabel}: ${rubric.description}`);
+    }
+    parts.push("", "For each rubric, determine if it is satisfied and provide brief reasoning.");
+    return parts.join("\n");
+  }
+  calculateScore(result, rubrics) {
+    const rubricMap = new Map(rubrics.map((r) => [r.id, r]));
+    const hits = [];
+    const misses = [];
+    let totalWeight = 0;
+    let earnedWeight = 0;
+    let failedRequired = false;
+    for (const check of result.checks) {
+      const rubric = rubricMap.get(check.id);
+      if (!rubric) {
+        continue;
+      }
+      totalWeight += rubric.weight;
+      if (check.satisfied) {
+        earnedWeight += rubric.weight;
+        hits.push(`[${rubric.id}] ${rubric.description}: ${check.reasoning}`);
+      } else {
+        misses.push(`[${rubric.id}] ${rubric.description}: ${check.reasoning}`);
+        if (rubric.required) {
+          failedRequired = true;
+        }
+      }
+    }
+    const score = totalWeight > 0 ? Math.min(1, Math.max(0, earnedWeight / totalWeight)) : 0;
+    let verdict;
+    if (failedRequired) {
+      verdict = "fail";
+    } else if (score >= 0.8) {
+      verdict = "pass";
+    } else if (score >= 0.6) {
+      verdict = "borderline";
+    } else {
+      verdict = "fail";
+    }
+    return { score, verdict, hits, misses };
+  }
+};
+
 // src/evaluation/evaluators.ts
 var DEFAULT_EVALUATOR_TEMPLATE = `You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.
 
@@ -3833,6 +4034,7 @@ async function runEvaluatorList(options) {
           name: evaluator.name,
           type: evaluator.type,
           score: score2.score,
+          verdict: score2.verdict,
           hits: score2.hits,
           misses: score2.misses,
           reasoning: score2.reasoning,
@@ -3860,6 +4062,40 @@ async function runEvaluatorList(options) {
           name: evaluator.name,
           type: evaluator.type,
           score: score2.score,
+          verdict: score2.verdict,
+          hits: score2.hits,
+          misses: score2.misses,
+          reasoning: score2.reasoning,
+          evaluator_provider_request: score2.evaluatorRawRequest
+        });
+        continue;
+      }
+      if (evaluator.type === "rubric") {
+        const rubricEvaluator = new RubricEvaluator({
+          config: evaluator,
+          resolveJudgeProvider: async (context) => {
+            if (context.judgeProvider) {
+              return context.judgeProvider;
+            }
+            return judgeProvider;
+          }
+        });
+        const score2 = await rubricEvaluator.evaluate({
+          evalCase,
+          candidate,
+          target,
+          provider,
+          attempt,
+          promptInputs,
+          now,
+          judgeProvider
+        });
+        scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
+        evaluatorResults.push({
+          name: evaluator.name,
+          type: evaluator.type,
+          score: score2.score,
+          verdict: score2.verdict,
           hits: score2.hits,
           misses: score2.misses,
           reasoning: score2.reasoning,
@@ -4090,6 +4326,86 @@ function isTimeoutLike(error) {
   return value.includes("timeout");
 }
 
+// src/evaluation/generators/rubric-generator.ts
+import { generateText as generateText3 } from "ai";
+import { z as z2 } from "zod";
+var rubricItemSchema = z2.object({
+  id: z2.string().describe("Short identifier for this rubric (e.g., clarity, completeness)"),
+  description: z2.string().describe("What this rubric checks for"),
+  weight: z2.number().default(1).describe("Relative importance (default 1.0)"),
+  required: z2.boolean().default(true).describe("Whether this is a mandatory requirement")
+});
+var rubricGenerationSchema = z2.object({
+  rubrics: z2.array(rubricItemSchema).describe("List of evaluation rubrics")
+});
+async function generateRubrics(options) {
+  const { expectedOutcome, question, referenceAnswer, provider } = options;
+  const prompt = buildPrompt(expectedOutcome, question, referenceAnswer);
+  const model = provider.asLanguageModel?.();
+  if (!model) {
+    throw new Error("Provider does not support language model interface");
+  }
+  const system = `You are an expert at creating evaluation rubrics.
+You must return a valid JSON object matching this schema:
+{
+  "rubrics": [
+    {
+      "id": "string (short identifier)",
+      "description": "string (what to check)",
+      "weight": number (default 1.0),
+      "required": boolean (default true)
+    }
+  ]
+}`;
+  let result;
+  let lastError;
+  for (let attempt = 1; attempt <= 3; attempt++) {
+    try {
+      const { text } = await generateText3({
+        model,
+        system,
+        prompt
+      });
+      const cleaned = text.replace(/```json\n?|```/g, "").trim();
+      result = rubricGenerationSchema.parse(JSON.parse(cleaned));
+      break;
+    } catch (e) {
+      lastError = e instanceof Error ? e : new Error(String(e));
+    }
+  }
+  if (!result) {
+    throw new Error(`Failed to parse generated rubrics after 3 attempts: ${lastError?.message}`);
+  }
+  return result.rubrics;
+}
+function buildPrompt(expectedOutcome, question, referenceAnswer) {
+  const parts = [
+    "You are an expert at creating evaluation rubrics.",
+    "Given the expected outcome (and optionally the question and reference answer),",
+    "generate a list of specific, measurable rubric items to evaluate whether an answer meets the expected outcome.",
+    "",
+    "Each rubric should:",
+    "- Be specific and testable",
+    "- Have a short, descriptive ID",
+    "- Include a clear description of what to check",
+    "- Indicate if it is required (mandatory) or optional",
+    "- Have an appropriate weight (default 1.0, use higher values for more important aspects)",
+    "",
+    "Generate 3-7 rubric items that comprehensively cover the expected outcome.",
+    "",
+    "[[ ## expected_outcome ## ]]",
+    expectedOutcome,
+    ""
+  ];
+  if (question && question.trim().length > 0) {
+    parts.push("[[ ## question ## ]]", question, "");
+  }
+  if (referenceAnswer && referenceAnswer.trim().length > 0) {
+    parts.push("[[ ## reference_answer ## ]]", referenceAnswer, "");
+  }
+  return parts.join("\n");
+}
+
 // src/index.ts
 function createAgentKernel() {
   return { status: "stub" };
@@ -4097,6 +4413,7 @@ function createAgentKernel() {
 export {
   CodeEvaluator,
   LlmJudgeEvaluator,
+  RubricEvaluator,
   TEST_MESSAGE_ROLES,
   buildDirectoryChain,
   buildPromptInputs,
@@ -4108,6 +4425,7 @@ export {
   extractCodeBlocks,
   fileExists,
   findGitRoot,
+  generateRubrics,
   getHitCount,
   isEvaluatorKind,
   isGuidelineFile,