@agentv/core 0.22.1 → 0.22.2

package/dist/index.js

@@ -453,25 +453,25 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
       }
     }
     const _model = asString2(rawEvaluator.model);
+    const rawRubrics = rawEvaluator.rubrics;
+    const parsedRubrics = Array.isArray(rawRubrics) ? rawRubrics.filter((r) => isJsonObject2(r)).map((rubric, index) => ({
+      id: asString2(rubric.id) ?? `rubric-${index + 1}`,
+      description: asString2(rubric.description) ?? "",
+      weight: typeof rubric.weight === "number" ? rubric.weight : 1,
+      required: typeof rubric.required === "boolean" ? rubric.required : true
+    })).filter((r) => r.description.length > 0) : void 0;
     if (typeValue === "rubric") {
-      const rubrics = rawEvaluator.rubrics;
-      if (!Array.isArray(rubrics)) {
+      if (!parsedRubrics) {
         logWarning2(`Skipping rubric evaluator '${name}' in '${evalId}': missing rubrics array`);
         continue;
       }
-      const parsedRubrics = rubrics.filter((r) => isJsonObject2(r)).map((rubric, index) => ({
-        id: asString2(rubric.id) ?? `rubric-${index + 1}`,
-        description: asString2(rubric.description) ?? "",
-        weight: typeof rubric.weight === "number" ? rubric.weight : 1,
-        required: typeof rubric.required === "boolean" ? rubric.required : true
-      })).filter((r) => r.description.length > 0);
       if (parsedRubrics.length === 0) {
         logWarning2(`Skipping rubric evaluator '${name}' in '${evalId}': no valid rubrics found`);
         continue;
       }
       evaluators.push({
         name,
-        type: "rubric",
+        type: "llm_judge",
         rubrics: parsedRubrics
       });
       continue;
@@ -480,7 +480,8 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
       name,
       type: "llm_judge",
       prompt,
-      promptPath
+      promptPath,
+      ...parsedRubrics && parsedRubrics.length > 0 ? { rubrics: parsedRubrics } : {}
     });
   }
   return evaluators.length > 0 ? evaluators : void 0;
@@ -1031,7 +1032,7 @@ async function loadEvalCases(evalFilePath, repoRoot, options) {
       if (rubricItems.length > 0) {
         const rubricEvaluator = {
           name: "rubric",
-          type: "rubric",
+          type: "llm_judge",
           rubrics: rubricItems
         };
         evaluators = evaluators ? [rubricEvaluator, ...evaluators] : [rubricEvaluator];
@@ -2928,149 +2929,9 @@ function resolveAndCreateProvider(definition, env = process.env) {
   return createProvider(resolved);
 }
 
-// src/evaluation/evaluators/rubric-evaluator.ts
+// src/evaluation/evaluators.ts
 import { generateText as generateText2 } from "ai";
 import { z } from "zod";
-var rubricCheckResultSchema = z.object({
-  id: z.string().describe("The ID of the rubric item being checked"),
-  satisfied: z.boolean().describe("Whether this rubric requirement is met"),
-  reasoning: z.string().describe("Brief explanation (1-2 sentences) for this check")
-});
-var rubricEvaluationSchema = z.object({
-  checks: z.array(rubricCheckResultSchema).describe("Results for each rubric item"),
-  overall_reasoning: z.string().describe("Overall assessment summary (1-2 sentences)")
-});
-var RubricEvaluator = class {
-  kind = "rubric";
-  config;
-  resolveJudgeProvider;
-  constructor(options) {
-    this.config = options.config;
-    this.resolveJudgeProvider = options.resolveJudgeProvider;
-  }
-  async evaluate(context) {
-    const judgeProvider = await this.resolveJudgeProvider(context);
-    if (!judgeProvider) {
-      throw new Error("No judge provider available for rubric evaluation");
-    }
-    if (!this.config.rubrics || this.config.rubrics.length === 0) {
-      throw new Error(
-        `No rubrics found for evaluator "${this.config.name}". Run "agentv generate rubrics" first.`
-      );
-    }
-    const prompt = this.buildPrompt(context, this.config.rubrics);
-    const model = judgeProvider.asLanguageModel?.();
-    if (!model) {
-      throw new Error("Judge provider does not support language model interface");
-    }
-    const system = `You are an expert evaluator. Evaluate the candidate answer against each rubric item.
-You must return a valid JSON object matching this schema:
-{
-  "checks": [
-    {
-      "id": "string (rubric id)",
-      "satisfied": boolean,
-      "reasoning": "string (brief explanation)"
-    }
-  ],
-  "overall_reasoning": "string (summary)"
-}`;
-    let result;
-    let lastError;
-    for (let attempt = 1; attempt <= 3; attempt++) {
-      try {
-        const { text } = await generateText2({
-          model,
-          system,
-          prompt
-        });
-        const cleaned = text.replace(/```json\n?|```/g, "").trim();
-        result = rubricEvaluationSchema.parse(JSON.parse(cleaned));
-        break;
-      } catch (e) {
-        lastError = e instanceof Error ? e : new Error(String(e));
-      }
-    }
-    if (!result) {
-      throw new Error(
-        `Failed to parse rubric evaluation result after 3 attempts: ${lastError?.message}`
-      );
-    }
-    const { score, verdict, hits, misses } = this.calculateScore(result, this.config.rubrics);
-    return {
-      score,
-      verdict,
-      hits,
-      misses,
-      expectedAspectCount: this.config.rubrics.length,
-      reasoning: result.overall_reasoning,
-      evaluatorRawRequest: {
-        prompt
-      }
-    };
-  }
-  buildPrompt(context, rubrics) {
-    const parts = [
-      "You are an expert evaluator. Evaluate the candidate answer against each rubric item below.",
-      "",
-      "[[ ## question ## ]]",
-      context.evalCase.question,
-      "",
-      "[[ ## expected_outcome ## ]]",
-      context.evalCase.expected_outcome,
-      ""
-    ];
-    if (context.evalCase.reference_answer && context.evalCase.reference_answer.trim().length > 0) {
-      parts.push("[[ ## reference_answer ## ]]", context.evalCase.reference_answer, "");
-    }
-    parts.push("[[ ## candidate_answer ## ]]", context.candidate, "", "[[ ## rubrics ## ]]");
-    for (const rubric of rubrics) {
-      const requiredLabel = rubric.required ? " (REQUIRED)" : "";
-      const weightLabel = rubric.weight !== 1 ? ` (weight: ${rubric.weight})` : "";
-      parts.push(`- [${rubric.id}]${requiredLabel}${weightLabel}: ${rubric.description}`);
-    }
-    parts.push("", "For each rubric, determine if it is satisfied and provide brief reasoning.");
-    return parts.join("\n");
-  }
-  calculateScore(result, rubrics) {
-    const rubricMap = new Map(rubrics.map((r) => [r.id, r]));
-    const hits = [];
-    const misses = [];
-    let totalWeight = 0;
-    let earnedWeight = 0;
-    let failedRequired = false;
-    for (const check of result.checks) {
-      const rubric = rubricMap.get(check.id);
-      if (!rubric) {
-        continue;
-      }
-      totalWeight += rubric.weight;
-      if (check.satisfied) {
-        earnedWeight += rubric.weight;
-        hits.push(`[${rubric.id}] ${rubric.description}: ${check.reasoning}`);
-      } else {
-        misses.push(`[${rubric.id}] ${rubric.description}: ${check.reasoning}`);
-        if (rubric.required) {
-          failedRequired = true;
-        }
-      }
-    }
-    const score = totalWeight > 0 ? Math.min(1, Math.max(0, earnedWeight / totalWeight)) : 0;
-    let verdict;
-    if (failedRequired) {
-      verdict = "fail";
-    } else if (score >= 0.8) {
-      verdict = "pass";
-    } else if (score >= 0.6) {
-      verdict = "borderline";
-    } else {
-      verdict = "fail";
-    }
-    return { score, verdict, hits, misses };
-  }
-};
-
-// src/evaluation/evaluators.ts
 var DEFAULT_EVALUATOR_TEMPLATE = `You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.
 
 Use the reference_answer as a gold standard for a high-quality response (if provided). The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.
@@ -3088,6 +2949,21 @@ Be concise and focused in your evaluation. Provide succinct, specific feedback r
 
 [[ ## candidate_answer ## ]]
 {{${TEMPLATE_VARIABLES.CANDIDATE_ANSWER}}}`;
+var freeformEvaluationSchema = z.object({
+  score: z.number().min(0).max(1).describe("Score between 0.0 and 1.0"),
+  hits: z.array(z.string()).describe("Brief specific achievements").optional(),
+  misses: z.array(z.string()).describe("Brief failures or omissions").optional(),
+  reasoning: z.string().describe("Concise explanation (1-2 sentences)").optional()
+});
+var rubricCheckResultSchema = z.object({
+  id: z.string().describe("The ID of the rubric item being checked"),
+  satisfied: z.boolean().describe("Whether this rubric requirement is met"),
+  reasoning: z.string().describe("Brief explanation (1-2 sentences) for this check")
+});
+var rubricEvaluationSchema = z.object({
+  checks: z.array(rubricCheckResultSchema).describe("Results for each rubric item"),
+  overall_reasoning: z.string().describe("Overall assessment summary (1-2 sentences)")
+});
 var LlmJudgeEvaluator = class {
   kind = "llm_judge";
   resolveJudgeProvider;
@@ -3105,9 +2981,13 @@ var LlmJudgeEvaluator = class {
     if (!judgeProvider) {
       throw new Error("No judge provider available for LLM grading");
     }
-    return this.evaluateWithPrompt(context, judgeProvider);
+    const config = context.evaluator;
+    if (config?.type === "llm_judge" && config.rubrics && config.rubrics.length > 0) {
+      return this.evaluateWithRubrics(context, judgeProvider, config.rubrics);
+    }
+    return this.evaluateFreeform(context, judgeProvider);
   }
-  async evaluateWithPrompt(context, judgeProvider) {
+  async evaluateFreeform(context, judgeProvider) {
     const formattedQuestion = context.promptInputs.question && context.promptInputs.question.trim().length > 0 ? context.promptInputs.question : context.evalCase.question;
     const variables = {
       [TEMPLATE_VARIABLES.INPUT_MESSAGES]: JSON.stringify(context.evalCase.input_segments, null, 2),
@@ -3124,34 +3004,132 @@ var LlmJudgeEvaluator = class {
     const systemPrompt = buildOutputSchema();
     const evaluatorTemplate = context.evaluatorTemplateOverride ?? this.evaluatorTemplate ?? DEFAULT_EVALUATOR_TEMPLATE;
     const userPrompt = substituteVariables(evaluatorTemplate, variables);
-    const response = await judgeProvider.invoke({
-      question: userPrompt,
-      systemPrompt,
-      evalCaseId: context.evalCase.id,
-      attempt: context.attempt,
-      maxOutputTokens: this.maxOutputTokens,
-      temperature: this.temperature
-    });
-    const parsed = parseQualityResponse(response);
-    const score = clampScore(parsed.score ?? 0);
-    const hits = Array.isArray(parsed.hits) ? parsed.hits.filter(isNonEmptyString).slice(0, 4) : [];
-    const misses = Array.isArray(parsed.misses) ? parsed.misses.filter(isNonEmptyString).slice(0, 4) : [];
-    const reasoning = parsed.reasoning ?? response.reasoning;
-    const expectedAspectCount = Math.max(hits.length + misses.length, 1);
     const evaluatorRawRequest = {
       userPrompt,
       systemPrompt,
       target: judgeProvider.targetName
     };
+    try {
+      const { data, providerResponse } = await this.runWithRetry({
+        context,
+        judgeProvider,
+        systemPrompt,
+        userPrompt,
+        schema: freeformEvaluationSchema
+      });
+      const score = clampScore(data.score);
+      const hits = Array.isArray(data.hits) ? data.hits.filter(isNonEmptyString).slice(0, 4) : [];
+      const misses = Array.isArray(data.misses) ? data.misses.filter(isNonEmptyString).slice(0, 4) : [];
+      const reasoning = data.reasoning ?? providerResponse?.reasoning;
+      const expectedAspectCount = Math.max(hits.length + misses.length, 1);
+      return {
+        score,
+        verdict: scoreToVerdict(score),
+        hits,
+        misses,
+        expectedAspectCount,
+        reasoning,
+        evaluatorRawRequest
+      };
+    } catch {
+      return {
+        score: 0,
+        verdict: "fail",
+        hits: [],
+        misses: [],
+        expectedAspectCount: 1,
+        evaluatorRawRequest
+      };
+    }
+  }
+  async evaluateWithRubrics(context, judgeProvider, rubrics) {
+    if (!rubrics || rubrics.length === 0) {
+      throw new Error(
+        `No rubrics found for evaluator "${context.evaluator?.name ?? "llm_judge"}". Run "agentv generate rubrics" first.`
+      );
+    }
+    const prompt = this.buildRubricPrompt(context, rubrics);
+    const systemPrompt = buildRubricOutputSchema();
+    const evaluatorRawRequest = {
+      userPrompt: prompt,
+      systemPrompt,
+      target: judgeProvider.targetName
+    };
+    const { data } = await this.runWithRetry({
+      context,
+      judgeProvider,
+      systemPrompt,
+      userPrompt: prompt,
+      schema: rubricEvaluationSchema
+    });
+    const { score, verdict, hits, misses } = calculateRubricScore(data, rubrics);
     return {
       score,
+      verdict,
       hits,
       misses,
-      expectedAspectCount,
-      reasoning,
+      expectedAspectCount: rubrics.length,
+      reasoning: data.overall_reasoning,
       evaluatorRawRequest
     };
   }
+  buildRubricPrompt(context, rubrics) {
+    const formattedQuestion = context.promptInputs.question && context.promptInputs.question.trim().length > 0 ? context.promptInputs.question : context.evalCase.question;
+    const parts = [
+      "You are an expert evaluator. Evaluate the candidate answer against each rubric item below.",
+      "",
+      "[[ ## question ## ]]",
+      formattedQuestion,
+      "",
+      "[[ ## expected_outcome ## ]]",
+      context.evalCase.expected_outcome,
+      ""
+    ];
+    if (context.evalCase.reference_answer && context.evalCase.reference_answer.trim().length > 0) {
+      parts.push("[[ ## reference_answer ## ]]", context.evalCase.reference_answer, "");
+    }
+    parts.push("[[ ## candidate_answer ## ]]", context.candidate, "", "[[ ## rubrics ## ]]");
+    for (const rubric of rubrics) {
+      const requiredLabel = rubric.required ? " (REQUIRED)" : "";
+      const weightLabel = rubric.weight !== 1 ? ` (weight: ${rubric.weight})` : "";
+      parts.push(`- [${rubric.id}]${requiredLabel}${weightLabel}: ${rubric.description}`);
+    }
+    parts.push("", "For each rubric, determine if it is satisfied and provide brief reasoning.");
+    return parts.join("\n");
+  }
+  async runWithRetry(options) {
+    const { context, judgeProvider, systemPrompt, userPrompt, schema } = options;
+    let lastError;
+    for (let attempt = 1; attempt <= 3; attempt++) {
+      try {
+        const model = judgeProvider.asLanguageModel?.();
+        if (model) {
+          const { text } = await generateText2({
+            model,
+            system: systemPrompt,
+            prompt: userPrompt,
+            ...this.maxOutputTokens ? { maxTokens: this.maxOutputTokens } : {},
+            ...typeof this.temperature === "number" ? { temperature: this.temperature } : {}
+          });
+          const data2 = schema.parse(parseJsonFromText(text));
+          return { data: data2 };
+        }
+        const response = await judgeProvider.invoke({
+          question: userPrompt,
+          systemPrompt,
+          evalCaseId: context.evalCase.id,
+          attempt: context.attempt,
+          maxOutputTokens: this.maxOutputTokens,
+          temperature: this.temperature
+        });
+        const data = schema.parse(parseJsonFromText(response.text ?? ""));
+        return { data, providerResponse: response };
+      } catch (e) {
+        lastError = e instanceof Error ? e : new Error(String(e));
+      }
+    }
+    throw new Error(`Failed to parse evaluator response after 3 attempts: ${lastError?.message}`);
+  }
 };
 function buildOutputSchema() {
   return [
@@ -3165,6 +3143,29 @@ function buildOutputSchema() {
     "}"
   ].join("\n");
 }
+function buildRubricOutputSchema() {
+  return `You are an expert evaluator. Evaluate the candidate answer against each rubric item.
+You must return a valid JSON object matching this schema:
+{
+  "checks": [
+    {
+      "id": "string (rubric id)",
+      "satisfied": boolean,
+      "reasoning": "string (brief explanation)"
+    }
+  ],
+  "overall_reasoning": "string (summary)"
+}`;
+}
+function scoreToVerdict(score) {
+  if (score >= 0.8) {
+    return "pass";
+  }
+  if (score >= 0.6) {
+    return "borderline";
+  }
+  return "fail";
+}
 function clampScore(value) {
   if (Number.isNaN(value) || !Number.isFinite(value)) {
     return 0;
@@ -3177,71 +3178,15 @@ function clampScore(value) {
   }
   return value;
 }
-function parseQualityResponse(response) {
-  const text = typeof response.text === "string" ? response.text.trim() : "";
-  if (text.length === 0) {
-    return {};
-  }
-  const direct = attemptParseJson(text);
-  if (direct && validateQualityJson(direct)) {
-    return direct;
-  }
-  const extracted = extractJsonBlob(text);
-  if (extracted) {
-    const parsed = attemptParseJson(extracted);
-    if (parsed && validateQualityJson(parsed)) {
-      return parsed;
-    }
-  }
-  return {};
-}
-function attemptParseJson(text) {
-  try {
-    const parsed = JSON.parse(text);
-    const score = typeof parsed.score === "number" ? parsed.score : void 0;
-    const hits = parsed.hits;
-    const misses = parsed.misses;
-    const reasoning = typeof parsed.reasoning === "string" ? parsed.reasoning : void 0;
-    return { score, hits, misses, reasoning };
-  } catch {
-    return void 0;
-  }
-}
-function validateQualityJson(parsed) {
-  if (typeof parsed.score !== "number") {
-    return false;
-  }
-  if (Number.isNaN(parsed.score) || !Number.isFinite(parsed.score)) {
-    return false;
-  }
-  if (parsed.score < 0 || parsed.score > 1) {
-    return false;
-  }
-  if (parsed.hits !== void 0) {
-    if (!Array.isArray(parsed.hits)) {
-      return false;
-    }
-    if (!parsed.hits.every((item) => typeof item === "string")) {
-      return false;
-    }
-  }
-  if (parsed.misses !== void 0) {
-    if (!Array.isArray(parsed.misses)) {
-      return false;
-    }
-    if (!parsed.misses.every((item) => typeof item === "string")) {
-      return false;
-    }
-  }
-  if (parsed.reasoning !== void 0 && typeof parsed.reasoning !== "string") {
-    return false;
-  }
-  return true;
-}
 function extractJsonBlob(text) {
   const match = text.match(/\{[\s\S]*\}/);
   return match?.[0];
 }
+function parseJsonFromText(text) {
+  const cleaned = typeof text === "string" ? text.replace(/```json\n?|```/g, "").trim() : "";
+  const blob = extractJsonBlob(cleaned) ?? cleaned;
+  return JSON.parse(blob);
+}
 function isNonEmptyString(value) {
   return typeof value === "string" && value.trim().length > 0;
 }
@@ -3278,6 +3223,7 @@ var CodeEvaluator = class {
       const reasoning = typeof parsed?.reasoning === "string" ? parsed.reasoning : void 0;
       return {
         score,
+        verdict: scoreToVerdict(score),
         hits,
         misses,
         expectedAspectCount: hits.length + misses.length || 1,
@@ -3291,6 +3237,7 @@ var CodeEvaluator = class {
       const message = error instanceof Error ? error.message : String(error);
       return {
         score: 0,
+        verdict: "fail",
         hits: [],
         misses: [`Code evaluator failed: ${message}`],
         expectedAspectCount: 1,
@@ -3304,6 +3251,33 @@ var CodeEvaluator = class {
     }
   }
 };
+function calculateRubricScore(result, rubrics) {
+  const rubricMap = new Map(rubrics.map((rubric) => [rubric.id, rubric]));
+  const hits = [];
+  const misses = [];
+  let totalWeight = 0;
+  let earnedWeight = 0;
+  let failedRequired = false;
+  for (const check of result.checks) {
+    const rubric = rubricMap.get(check.id);
+    if (!rubric) {
+      continue;
+    }
+    totalWeight += rubric.weight;
+    if (check.satisfied) {
+      earnedWeight += rubric.weight;
+      hits.push(`[${rubric.id}] ${rubric.description}: ${check.reasoning}`);
+    } else {
+      misses.push(`[${rubric.id}] ${rubric.description}: ${check.reasoning}`);
+      if (rubric.required) {
+        failedRequired = true;
+      }
+    }
+  }
+  const score = totalWeight > 0 ? Math.min(1, Math.max(0, earnedWeight / totalWeight)) : 0;
+  const verdict = failedRequired ? "fail" : scoreToVerdict(score);
+  return { score, verdict, hits, misses };
+}
 async function executeScript(scriptPath, input, agentTimeoutMs, cwd) {
   const { spawn: spawn2 } = await import("node:child_process");
   return await new Promise((resolve, reject) => {
@@ -4040,7 +4014,6 @@ async function runEvaluatorList(options) {
           reasoning: score2.reasoning,
           evaluator_provider_request: score2.evaluatorRawRequest
         });
-        continue;
       }
       if (evaluator.type === "code") {
         const codeEvaluator = new CodeEvaluator({
@@ -4068,44 +4041,12 @@ async function runEvaluatorList(options) {
           reasoning: score2.reasoning,
           evaluator_provider_request: score2.evaluatorRawRequest
         });
-        continue;
-      }
-      if (evaluator.type === "rubric") {
-        const rubricEvaluator = new RubricEvaluator({
-          config: evaluator,
-          resolveJudgeProvider: async (context) => {
-            if (context.judgeProvider) {
-              return context.judgeProvider;
-            }
-            return judgeProvider;
-          }
-        });
-        const score2 = await rubricEvaluator.evaluate({
-          evalCase,
-          candidate,
-          target,
-          provider,
-          attempt,
-          promptInputs,
-          now,
-          judgeProvider
-        });
-        scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
-        evaluatorResults.push({
-          name: evaluator.name,
-          type: evaluator.type,
-          score: score2.score,
-          verdict: score2.verdict,
-          hits: score2.hits,
-          misses: score2.misses,
-          reasoning: score2.reasoning,
-          evaluator_provider_request: score2.evaluatorRawRequest
-        });
       }
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       const fallbackScore = {
         score: 0,
+        verdict: "fail",
         hits: [],
         misses: [`Evaluator '${evaluator.name}' failed: ${message}`],
         expectedAspectCount: 1,
@@ -4120,6 +4061,7 @@ async function runEvaluatorList(options) {
         name: evaluator.name ?? "unknown",
         type: evaluator.type ?? "unknown",
         score: 0,
+        verdict: "fail",
         hits: [],
         misses: [`Evaluator '${evaluator.name ?? "unknown"}' failed: ${message}`],
         reasoning: message
@@ -4138,6 +4080,7 @@ async function runEvaluatorList(options) {
   const reasoning = reasoningParts.length > 0 ? reasoningParts.join(" | ") : void 0;
   const score = {
     score: aggregateScore,
+    verdict: scoreToVerdict2(aggregateScore),
     hits,
     misses,
     expectedAspectCount,
@@ -4188,6 +4131,15 @@ async function resolveCustomPrompt(config) {
 function isNonEmptyString2(value) {
   return typeof value === "string" && value.trim().length > 0;
 }
+function scoreToVerdict2(score) {
+  if (score >= 0.8) {
+    return "pass";
+  }
+  if (score >= 0.6) {
+    return "borderline";
+  }
+  return "fail";
+}
 function filterEvalCases(evalCases, evalId) {
   if (!evalId) {
     return evalCases;
@@ -4413,7 +4365,6 @@ function createAgentKernel() {
 export {
   CodeEvaluator,
   LlmJudgeEvaluator,
-  RubricEvaluator,
   TEST_MESSAGE_ROLES,
   buildDirectoryChain,
   buildPromptInputs,