@agentv/core 0.22.1 → 0.23.0

package/dist/index.cjs

@@ -31,8 +31,8 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var index_exports = {};
 __export(index_exports, {
   CodeEvaluator: () => CodeEvaluator,
+  CompositeEvaluator: () => CompositeEvaluator,
   LlmJudgeEvaluator: () => LlmJudgeEvaluator,
-  RubricEvaluator: () => RubricEvaluator,
   TEST_MESSAGE_ROLES: () => TEST_MESSAGE_ROLES,
   buildDirectoryChain: () => buildDirectoryChain2,
   buildPromptInputs: () => buildPromptInputs,
@@ -108,7 +108,7 @@ function isTestMessage(value) {
   }
   return candidate.content.every(isJsonObject);
 }
-var EVALUATOR_KIND_VALUES = ["code", "llm_judge", "rubric"];
+var EVALUATOR_KIND_VALUES = ["code_judge", "llm_judge", "rubric", "composite"];
 var EVALUATOR_KIND_SET = new Set(EVALUATOR_KIND_VALUES);
 function isEvaluatorKind(value) {
   return typeof value === "string" && EVALUATOR_KIND_SET.has(value);
@@ -460,10 +460,10 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
       logWarning2(`Skipping evaluator with invalid name/type in '${evalId}'`);
       continue;
     }
-    if (typeValue === "code") {
+    if (typeValue === "code_judge") {
       const script = asString2(rawEvaluator.script);
       if (!script) {
-        logWarning2(`Skipping code evaluator '${name}' in '${evalId}': missing script`);
+        logWarning2(`Skipping code_judge evaluator '${name}' in '${evalId}': missing script`);
         continue;
       }
       const cwd = asString2(rawEvaluator.cwd);
@@ -474,7 +474,7 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
           resolvedCwd = import_node_path3.default.resolve(resolved.resolvedPath);
         } else {
           logWarning2(
-            `Code evaluator '${name}' in '${evalId}': cwd not found (${resolved.displayPath})`,
+            `Code_judge evaluator '${name}' in '${evalId}': cwd not found (${resolved.displayPath})`,
             resolved.attempted.length > 0 ? resolved.attempted.map((attempt) => `  Tried: ${attempt}`) : void 0
           );
         }
@@ -490,6 +490,105 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
       });
       continue;
     }
+    if (typeValue === "composite") {
+      const rawMembers = rawEvaluator.evaluators;
+      if (!Array.isArray(rawMembers)) {
+        logWarning2(
+          `Skipping composite evaluator '${name}' in '${evalId}': missing evaluators array`
+        );
+        continue;
+      }
+      const rawAggregator = rawEvaluator.aggregator;
+      if (!isJsonObject2(rawAggregator)) {
+        logWarning2(`Skipping composite evaluator '${name}' in '${evalId}': missing aggregator`);
+        continue;
+      }
+      const aggregatorType = asString2(rawAggregator.type);
+      if (aggregatorType !== "weighted_average" && aggregatorType !== "code_judge" && aggregatorType !== "llm_judge") {
+        logWarning2(
+          `Skipping composite evaluator '${name}' in '${evalId}': invalid aggregator type '${aggregatorType}'`
+        );
+        continue;
+      }
+      const memberEvaluators = [];
+      for (const rawMember of rawMembers) {
+        if (!isJsonObject2(rawMember)) {
+          logWarning2(`Skipping invalid member evaluator in composite '${name}' (expected object)`);
+          continue;
+        }
+        const memberName = asString2(rawMember.name);
+        const memberType = rawMember.type;
+        if (!memberName || !isEvaluatorKind(memberType)) {
+          logWarning2(`Skipping member evaluator with invalid name/type in composite '${name}'`);
+          continue;
+        }
+        const memberConfigs = await parseEvaluators(
+          { evaluators: [rawMember] },
+          void 0,
+          searchRoots,
+          `${evalId}:${name}:${memberName}`
+        );
+        if (memberConfigs && memberConfigs.length > 0) {
+          memberEvaluators.push(memberConfigs[0]);
+        }
+      }
+      if (memberEvaluators.length === 0) {
+        logWarning2(
+          `Skipping composite evaluator '${name}' in '${evalId}': no valid member evaluators`
+        );
+        continue;
+      }
+      let aggregator;
+      if (aggregatorType === "weighted_average") {
+        const weights = isJsonObject2(rawAggregator.weights) ? rawAggregator.weights : void 0;
+        const parsedWeights = {};
+        if (weights) {
+          for (const [key, value] of Object.entries(weights)) {
+            if (typeof value === "number") {
+              parsedWeights[key] = value;
+            }
+          }
+        }
+        aggregator = {
+          type: "weighted_average",
+          ...Object.keys(parsedWeights).length > 0 ? { weights: parsedWeights } : {}
+        };
+      } else if (aggregatorType === "code_judge") {
+        const aggregatorPath = asString2(rawAggregator.path);
+        if (!aggregatorPath) {
+          logWarning2(
+            `Skipping composite evaluator '${name}' in '${evalId}': code_judge aggregator missing path`
+          );
+          continue;
+        }
+        aggregator = {
+          type: "code_judge",
+          path: aggregatorPath,
+          cwd: searchRoots[0]
+        };
+      } else {
+        const aggregatorPrompt = asString2(rawAggregator.prompt);
+        let promptPath2;
+        if (aggregatorPrompt) {
+          const resolved = await resolveFileReference(aggregatorPrompt, searchRoots);
+          if (resolved.resolvedPath) {
+            promptPath2 = import_node_path3.default.resolve(resolved.resolvedPath);
+          }
+        }
+        aggregator = {
+          type: "llm_judge",
+          ...aggregatorPrompt ? { prompt: aggregatorPrompt } : {},
+          ...promptPath2 ? { promptPath: promptPath2 } : {}
+        };
+      }
+      evaluators.push({
+        name,
+        type: "composite",
+        evaluators: memberEvaluators,
+        aggregator
+      });
+      continue;
+    }
     const prompt = asString2(rawEvaluator.prompt);
     let promptPath;
     if (prompt) {
@@ -510,25 +609,25 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
       }
     }
     const _model = asString2(rawEvaluator.model);
+    const rawRubrics = rawEvaluator.rubrics;
+    const parsedRubrics = Array.isArray(rawRubrics) ? rawRubrics.filter((r) => isJsonObject2(r)).map((rubric, index) => ({
+      id: asString2(rubric.id) ?? `rubric-${index + 1}`,
+      description: asString2(rubric.description) ?? "",
+      weight: typeof rubric.weight === "number" ? rubric.weight : 1,
+      required: typeof rubric.required === "boolean" ? rubric.required : true
+    })).filter((r) => r.description.length > 0) : void 0;
     if (typeValue === "rubric") {
-      const rubrics = rawEvaluator.rubrics;
-      if (!Array.isArray(rubrics)) {
+      if (!parsedRubrics) {
         logWarning2(`Skipping rubric evaluator '${name}' in '${evalId}': missing rubrics array`);
         continue;
       }
-      const parsedRubrics = rubrics.filter((r) => isJsonObject2(r)).map((rubric, index) => ({
-        id: asString2(rubric.id) ?? `rubric-${index + 1}`,
-        description: asString2(rubric.description) ?? "",
-        weight: typeof rubric.weight === "number" ? rubric.weight : 1,
-        required: typeof rubric.required === "boolean" ? rubric.required : true
-      })).filter((r) => r.description.length > 0);
       if (parsedRubrics.length === 0) {
         logWarning2(`Skipping rubric evaluator '${name}' in '${evalId}': no valid rubrics found`);
         continue;
       }
       evaluators.push({
         name,
-        type: "rubric",
+        type: "llm_judge",
         rubrics: parsedRubrics
       });
       continue;
@@ -537,7 +636,8 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
       name,
       type: "llm_judge",
       prompt,
-      promptPath
+      promptPath,
+      ...parsedRubrics && parsedRubrics.length > 0 ? { rubrics: parsedRubrics } : {}
     });
   }
   return evaluators.length > 0 ? evaluators : void 0;
@@ -1088,7 +1188,7 @@ async function loadEvalCases(evalFilePath, repoRoot, options) {
       if (rubricItems.length > 0) {
         const rubricEvaluator = {
           name: "rubric",
-          type: "rubric",
+          type: "llm_judge",
           rubrics: rubricItems
         };
         evaluators = evaluators ? [rubricEvaluator, ...evaluators] : [rubricEvaluator];
@@ -3621,149 +3721,9 @@ function resolveAndCreateProvider(definition, env = process.env) {
   return createProvider(resolved);
 }
 
-// src/evaluation/evaluators/rubric-evaluator.ts
+// src/evaluation/evaluators.ts
 var import_ai2 = require("ai");
 var import_zod2 = require("zod");
-var rubricCheckResultSchema = import_zod2.z.object({
-  id: import_zod2.z.string().describe("The ID of the rubric item being checked"),
-  satisfied: import_zod2.z.boolean().describe("Whether this rubric requirement is met"),
-  reasoning: import_zod2.z.string().describe("Brief explanation (1-2 sentences) for this check")
-});
-var rubricEvaluationSchema = import_zod2.z.object({
-  checks: import_zod2.z.array(rubricCheckResultSchema).describe("Results for each rubric item"),
-  overall_reasoning: import_zod2.z.string().describe("Overall assessment summary (1-2 sentences)")
-});
-var RubricEvaluator = class {
-  kind = "rubric";
-  config;
-  resolveJudgeProvider;
-  constructor(options) {
-    this.config = options.config;
-    this.resolveJudgeProvider = options.resolveJudgeProvider;
-  }
-  async evaluate(context) {
-    const judgeProvider = await this.resolveJudgeProvider(context);
-    if (!judgeProvider) {
-      throw new Error("No judge provider available for rubric evaluation");
-    }
-    if (!this.config.rubrics || this.config.rubrics.length === 0) {
-      throw new Error(
-        `No rubrics found for evaluator "${this.config.name}". Run "agentv generate rubrics" first.`
-      );
-    }
-    const prompt = this.buildPrompt(context, this.config.rubrics);
-    const model = judgeProvider.asLanguageModel?.();
-    if (!model) {
-      throw new Error("Judge provider does not support language model interface");
-    }
-    const system = `You are an expert evaluator. Evaluate the candidate answer against each rubric item.
-You must return a valid JSON object matching this schema:
-{
-  "checks": [
-    {
-      "id": "string (rubric id)",
-      "satisfied": boolean,
-      "reasoning": "string (brief explanation)"
-    }
-  ],
-  "overall_reasoning": "string (summary)"
-}`;
-    let result;
-    let lastError;
-    for (let attempt = 1; attempt <= 3; attempt++) {
-      try {
-        const { text } = await (0, import_ai2.generateText)({
-          model,
-          system,
-          prompt
-        });
-        const cleaned = text.replace(/```json\n?|```/g, "").trim();
-        result = rubricEvaluationSchema.parse(JSON.parse(cleaned));
-        break;
-      } catch (e) {
-        lastError = e instanceof Error ? e : new Error(String(e));
-      }
-    }
-    if (!result) {
-      throw new Error(
-        `Failed to parse rubric evaluation result after 3 attempts: ${lastError?.message}`
-      );
-    }
-    const { score, verdict, hits, misses } = this.calculateScore(result, this.config.rubrics);
-    return {
-      score,
-      verdict,
-      hits,
-      misses,
-      expectedAspectCount: this.config.rubrics.length,
-      reasoning: result.overall_reasoning,
-      evaluatorRawRequest: {
-        prompt
-      }
-    };
-  }
-  buildPrompt(context, rubrics) {
-    const parts = [
-      "You are an expert evaluator. Evaluate the candidate answer against each rubric item below.",
-      "",
-      "[[ ## question ## ]]",
-      context.evalCase.question,
-      "",
-      "[[ ## expected_outcome ## ]]",
-      context.evalCase.expected_outcome,
-      ""
-    ];
-    if (context.evalCase.reference_answer && context.evalCase.reference_answer.trim().length > 0) {
-      parts.push("[[ ## reference_answer ## ]]", context.evalCase.reference_answer, "");
-    }
-    parts.push("[[ ## candidate_answer ## ]]", context.candidate, "", "[[ ## rubrics ## ]]");
-    for (const rubric of rubrics) {
-      const requiredLabel = rubric.required ? " (REQUIRED)" : "";
-      const weightLabel = rubric.weight !== 1 ? ` (weight: ${rubric.weight})` : "";
-      parts.push(`- [${rubric.id}]${requiredLabel}${weightLabel}: ${rubric.description}`);
-    }
-    parts.push("", "For each rubric, determine if it is satisfied and provide brief reasoning.");
-    return parts.join("\n");
-  }
-  calculateScore(result, rubrics) {
-    const rubricMap = new Map(rubrics.map((r) => [r.id, r]));
-    const hits = [];
-    const misses = [];
-    let totalWeight = 0;
-    let earnedWeight = 0;
-    let failedRequired = false;
-    for (const check of result.checks) {
-      const rubric = rubricMap.get(check.id);
-      if (!rubric) {
-        continue;
-      }
-      totalWeight += rubric.weight;
-      if (check.satisfied) {
-        earnedWeight += rubric.weight;
-        hits.push(`[${rubric.id}] ${rubric.description}: ${check.reasoning}`);
-      } else {
-        misses.push(`[${rubric.id}] ${rubric.description}: ${check.reasoning}`);
-        if (rubric.required) {
-          failedRequired = true;
-        }
-      }
-    }
-    const score = totalWeight > 0 ? Math.min(1, Math.max(0, earnedWeight / totalWeight)) : 0;
-    let verdict;
-    if (failedRequired) {
-      verdict = "fail";
-    } else if (score >= 0.8) {
-      verdict = "pass";
-    } else if (score >= 0.6) {
-      verdict = "borderline";
-    } else {
-      verdict = "fail";
-    }
-    return { score, verdict, hits, misses };
-  }
-};
-
-// src/evaluation/evaluators.ts
 var DEFAULT_EVALUATOR_TEMPLATE = `You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.
 
 Use the reference_answer as a gold standard for a high-quality response (if provided). The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.
@@ -3781,6 +3741,21 @@ Be concise and focused in your evaluation. Provide succinct, specific feedback r
 
 [[ ## candidate_answer ## ]]
 {{${TEMPLATE_VARIABLES.CANDIDATE_ANSWER}}}`;
+var freeformEvaluationSchema = import_zod2.z.object({
+  score: import_zod2.z.number().min(0).max(1).describe("Score between 0.0 and 1.0"),
+  hits: import_zod2.z.array(import_zod2.z.string()).describe("Brief specific achievements").optional(),
+  misses: import_zod2.z.array(import_zod2.z.string()).describe("Brief failures or omissions").optional(),
+  reasoning: import_zod2.z.string().describe("Concise explanation (1-2 sentences)").optional()
+});
+var rubricCheckResultSchema = import_zod2.z.object({
+  id: import_zod2.z.string().describe("The ID of the rubric item being checked"),
+  satisfied: import_zod2.z.boolean().describe("Whether this rubric requirement is met"),
+  reasoning: import_zod2.z.string().describe("Brief explanation (1-2 sentences) for this check")
+});
+var rubricEvaluationSchema = import_zod2.z.object({
+  checks: import_zod2.z.array(rubricCheckResultSchema).describe("Results for each rubric item"),
+  overall_reasoning: import_zod2.z.string().describe("Overall assessment summary (1-2 sentences)")
+});
 var LlmJudgeEvaluator = class {
   kind = "llm_judge";
   resolveJudgeProvider;
@@ -3798,9 +3773,13 @@ var LlmJudgeEvaluator = class {
     if (!judgeProvider) {
       throw new Error("No judge provider available for LLM grading");
     }
-    return this.evaluateWithPrompt(context, judgeProvider);
+    const config = context.evaluator;
+    if (config?.type === "llm_judge" && config.rubrics && config.rubrics.length > 0) {
+      return this.evaluateWithRubrics(context, judgeProvider, config.rubrics);
+    }
+    return this.evaluateFreeform(context, judgeProvider);
   }
-  async evaluateWithPrompt(context, judgeProvider) {
+  async evaluateFreeform(context, judgeProvider) {
     const formattedQuestion = context.promptInputs.question && context.promptInputs.question.trim().length > 0 ? context.promptInputs.question : context.evalCase.question;
     const variables = {
       [TEMPLATE_VARIABLES.INPUT_MESSAGES]: JSON.stringify(context.evalCase.input_segments, null, 2),
@@ -3817,34 +3796,132 @@ var LlmJudgeEvaluator = class {
     const systemPrompt = buildOutputSchema();
     const evaluatorTemplate = context.evaluatorTemplateOverride ?? this.evaluatorTemplate ?? DEFAULT_EVALUATOR_TEMPLATE;
     const userPrompt = substituteVariables(evaluatorTemplate, variables);
-    const response = await judgeProvider.invoke({
-      question: userPrompt,
-      systemPrompt,
-      evalCaseId: context.evalCase.id,
-      attempt: context.attempt,
-      maxOutputTokens: this.maxOutputTokens,
-      temperature: this.temperature
-    });
-    const parsed = parseQualityResponse(response);
-    const score = clampScore(parsed.score ?? 0);
-    const hits = Array.isArray(parsed.hits) ? parsed.hits.filter(isNonEmptyString).slice(0, 4) : [];
-    const misses = Array.isArray(parsed.misses) ? parsed.misses.filter(isNonEmptyString).slice(0, 4) : [];
-    const reasoning = parsed.reasoning ?? response.reasoning;
-    const expectedAspectCount = Math.max(hits.length + misses.length, 1);
     const evaluatorRawRequest = {
       userPrompt,
       systemPrompt,
       target: judgeProvider.targetName
     };
+    try {
+      const { data, providerResponse } = await this.runWithRetry({
+        context,
+        judgeProvider,
+        systemPrompt,
+        userPrompt,
+        schema: freeformEvaluationSchema
+      });
+      const score = clampScore(data.score);
+      const hits = Array.isArray(data.hits) ? data.hits.filter(isNonEmptyString).slice(0, 4) : [];
+      const misses = Array.isArray(data.misses) ? data.misses.filter(isNonEmptyString).slice(0, 4) : [];
+      const reasoning = data.reasoning ?? providerResponse?.reasoning;
+      const expectedAspectCount = Math.max(hits.length + misses.length, 1);
+      return {
+        score,
+        verdict: scoreToVerdict(score),
+        hits,
+        misses,
+        expectedAspectCount,
+        reasoning,
+        evaluatorRawRequest
+      };
+    } catch {
+      return {
+        score: 0,
+        verdict: "fail",
+        hits: [],
+        misses: [],
+        expectedAspectCount: 1,
+        evaluatorRawRequest
+      };
+    }
+  }
+  async evaluateWithRubrics(context, judgeProvider, rubrics) {
+    if (!rubrics || rubrics.length === 0) {
+      throw new Error(
+        `No rubrics found for evaluator "${context.evaluator?.name ?? "llm_judge"}". Run "agentv generate rubrics" first.`
+      );
+    }
+    const prompt = this.buildRubricPrompt(context, rubrics);
+    const systemPrompt = buildRubricOutputSchema();
+    const evaluatorRawRequest = {
+      userPrompt: prompt,
+      systemPrompt,
+      target: judgeProvider.targetName
+    };
+    const { data } = await this.runWithRetry({
+      context,
+      judgeProvider,
+      systemPrompt,
+      userPrompt: prompt,
+      schema: rubricEvaluationSchema
+    });
+    const { score, verdict, hits, misses } = calculateRubricScore(data, rubrics);
     return {
       score,
+      verdict,
       hits,
       misses,
-      expectedAspectCount,
-      reasoning,
+      expectedAspectCount: rubrics.length,
+      reasoning: data.overall_reasoning,
       evaluatorRawRequest
     };
   }
+  buildRubricPrompt(context, rubrics) {
+    const formattedQuestion = context.promptInputs.question && context.promptInputs.question.trim().length > 0 ? context.promptInputs.question : context.evalCase.question;
+    const parts = [
+      "You are an expert evaluator. Evaluate the candidate answer against each rubric item below.",
+      "",
+      "[[ ## question ## ]]",
+      formattedQuestion,
+      "",
+      "[[ ## expected_outcome ## ]]",
+      context.evalCase.expected_outcome,
+      ""
+    ];
+    if (context.evalCase.reference_answer && context.evalCase.reference_answer.trim().length > 0) {
+      parts.push("[[ ## reference_answer ## ]]", context.evalCase.reference_answer, "");
+    }
+    parts.push("[[ ## candidate_answer ## ]]", context.candidate, "", "[[ ## rubrics ## ]]");
+    for (const rubric of rubrics) {
+      const requiredLabel = rubric.required ? " (REQUIRED)" : "";
+      const weightLabel = rubric.weight !== 1 ? ` (weight: ${rubric.weight})` : "";
+      parts.push(`- [${rubric.id}]${requiredLabel}${weightLabel}: ${rubric.description}`);
+    }
+    parts.push("", "For each rubric, determine if it is satisfied and provide brief reasoning.");
+    return parts.join("\n");
+  }
+  async runWithRetry(options) {
+    const { context, judgeProvider, systemPrompt, userPrompt, schema } = options;
+    let lastError;
+    for (let attempt = 1; attempt <= 3; attempt++) {
+      try {
+        const model = judgeProvider.asLanguageModel?.();
+        if (model) {
+          const { text } = await (0, import_ai2.generateText)({
+            model,
+            system: systemPrompt,
+            prompt: userPrompt,
+            ...this.maxOutputTokens ? { maxTokens: this.maxOutputTokens } : {},
+            ...typeof this.temperature === "number" ? { temperature: this.temperature } : {}
+          });
+          const data2 = schema.parse(parseJsonFromText(text));
+          return { data: data2 };
+        }
+        const response = await judgeProvider.invoke({
+          question: userPrompt,
+          systemPrompt,
+          evalCaseId: context.evalCase.id,
+          attempt: context.attempt,
+          maxOutputTokens: this.maxOutputTokens,
+          temperature: this.temperature
+        });
+        const data = schema.parse(parseJsonFromText(response.text ?? ""));
+        return { data, providerResponse: response };
+      } catch (e) {
+        lastError = e instanceof Error ? e : new Error(String(e));
+      }
+    }
+    throw new Error(`Failed to parse evaluator response after 3 attempts: ${lastError?.message}`);
+  }
 };
 function buildOutputSchema() {
   return [
@@ -3858,6 +3935,29 @@ function buildOutputSchema() {
     "}"
   ].join("\n");
 }
+function buildRubricOutputSchema() {
+  return `You are an expert evaluator. Evaluate the candidate answer against each rubric item.
+You must return a valid JSON object matching this schema:
+{
+  "checks": [
+    {
+      "id": "string (rubric id)",
+      "satisfied": boolean,
+      "reasoning": "string (brief explanation)"
+    }
+  ],
+  "overall_reasoning": "string (summary)"
+}`;
+}
+function scoreToVerdict(score) {
+  if (score >= 0.8) {
+    return "pass";
+  }
+  if (score >= 0.6) {
+    return "borderline";
+  }
+  return "fail";
+}
 function clampScore(value) {
   if (Number.isNaN(value) || !Number.isFinite(value)) {
     return 0;
@@ -3870,71 +3970,15 @@ function clampScore(value) {
   }
   return value;
 }
-function parseQualityResponse(response) {
-  const text = typeof response.text === "string" ? response.text.trim() : "";
-  if (text.length === 0) {
-    return {};
-  }
-  const direct = attemptParseJson(text);
-  if (direct && validateQualityJson(direct)) {
-    return direct;
-  }
-  const extracted = extractJsonBlob(text);
-  if (extracted) {
-    const parsed = attemptParseJson(extracted);
-    if (parsed && validateQualityJson(parsed)) {
-      return parsed;
-    }
-  }
-  return {};
-}
-function attemptParseJson(text) {
-  try {
-    const parsed = JSON.parse(text);
-    const score = typeof parsed.score === "number" ? parsed.score : void 0;
-    const hits = parsed.hits;
-    const misses = parsed.misses;
-    const reasoning = typeof parsed.reasoning === "string" ? parsed.reasoning : void 0;
-    return { score, hits, misses, reasoning };
-  } catch {
-    return void 0;
-  }
-}
-function validateQualityJson(parsed) {
-  if (typeof parsed.score !== "number") {
-    return false;
-  }
-  if (Number.isNaN(parsed.score) || !Number.isFinite(parsed.score)) {
-    return false;
-  }
-  if (parsed.score < 0 || parsed.score > 1) {
-    return false;
-  }
-  if (parsed.hits !== void 0) {
-    if (!Array.isArray(parsed.hits)) {
-      return false;
-    }
-    if (!parsed.hits.every((item) => typeof item === "string")) {
-      return false;
-    }
-  }
-  if (parsed.misses !== void 0) {
-    if (!Array.isArray(parsed.misses)) {
-      return false;
-    }
-    if (!parsed.misses.every((item) => typeof item === "string")) {
-      return false;
-    }
-  }
-  if (parsed.reasoning !== void 0 && typeof parsed.reasoning !== "string") {
-    return false;
-  }
-  return true;
-}
 function extractJsonBlob(text) {
   const match = text.match(/\{[\s\S]*\}/);
   return match?.[0];
 }
+function parseJsonFromText(text) {
+  const cleaned = typeof text === "string" ? text.replace(/```json\n?|```/g, "").trim() : "";
+  const blob = extractJsonBlob(cleaned) ?? cleaned;
+  return JSON.parse(blob);
+}
 function isNonEmptyString(value) {
   return typeof value === "string" && value.trim().length > 0;
 }
@@ -3971,6 +4015,7 @@ var CodeEvaluator = class {
       const reasoning = typeof parsed?.reasoning === "string" ? parsed.reasoning : void 0;
       return {
         score,
+        verdict: scoreToVerdict(score),
         hits,
         misses,
         expectedAspectCount: hits.length + misses.length || 1,
@@ -3984,6 +4029,7 @@ var CodeEvaluator = class {
       const message = error instanceof Error ? error.message : String(error);
       return {
         score: 0,
+        verdict: "fail",
         hits: [],
         misses: [`Code evaluator failed: ${message}`],
         expectedAspectCount: 1,
@@ -3997,6 +4043,33 @@ var CodeEvaluator = class {
     }
   }
 };
+function calculateRubricScore(result, rubrics) {
+  const rubricMap = new Map(rubrics.map((rubric) => [rubric.id, rubric]));
+  const hits = [];
+  const misses = [];
+  let totalWeight = 0;
+  let earnedWeight = 0;
+  let failedRequired = false;
+  for (const check of result.checks) {
+    const rubric = rubricMap.get(check.id);
+    if (!rubric) {
+      continue;
+    }
+    totalWeight += rubric.weight;
+    if (check.satisfied) {
+      earnedWeight += rubric.weight;
+      hits.push(`[${rubric.id}] ${rubric.description}: ${check.reasoning}`);
+    } else {
+      misses.push(`[${rubric.id}] ${rubric.description}: ${check.reasoning}`);
+      if (rubric.required) {
+        failedRequired = true;
+      }
+    }
+  }
+  const score = totalWeight > 0 ? Math.min(1, Math.max(0, earnedWeight / totalWeight)) : 0;
+  const verdict = failedRequired ? "fail" : scoreToVerdict(score);
+  return { score, verdict, hits, misses };
+}
 async function executeScript(scriptPath, input, agentTimeoutMs, cwd) {
   const { spawn: spawn2 } = await import("child_process");
   return await new Promise((resolve, reject) => {
@@ -4048,6 +4121,228 @@ function substituteVariables(template, variables) {
     return variables[varName] ?? match;
   });
 }
+var DEFAULT_COMPOSITE_AGGREGATOR_PROMPT = `Review the following evaluation results:
+{{EVALUATOR_RESULTS_JSON}}
+
+Decide the final score and verdict based on all evaluator results.
+Return a JSON object with: score (0.0-1.0), verdict (pass/fail/borderline), and reasoning.`;
+var CompositeEvaluator = class {
+  kind = "composite";
+  config;
+  evaluatorFactory;
+  cwd;
+  constructor(options) {
+    this.config = options.config;
+    this.evaluatorFactory = options.evaluatorFactory;
+    this.cwd = options.cwd;
+  }
+  async evaluate(context) {
+    const memberResults = await Promise.all(
+      this.config.evaluators.map(async (memberConfig) => {
+        const evaluator = this.evaluatorFactory.create(memberConfig, context);
+        return {
+          id: memberConfig.name,
+          type: memberConfig.type,
+          result: await evaluator.evaluate(context)
+        };
+      })
+    );
+    return this.aggregate(memberResults, context);
+  }
+  async aggregate(results, context) {
+    const aggregator = this.config.aggregator;
+    switch (aggregator.type) {
+      case "code_judge":
+        return this.runCodeAggregator(results, aggregator.path, aggregator.cwd ?? this.cwd);
+      case "llm_judge":
+        return this.runLlmAggregator(results, context, aggregator);
+      default:
+        return this.runWeightedAverage(results, aggregator.weights);
+    }
+  }
+  runWeightedAverage(results, weights) {
+    let totalWeight = 0;
+    let weightedSum = 0;
+    const allHits = [];
+    const allMisses = [];
+    const reasoningParts = [];
+    const evaluatorResults = [];
+    for (const member of results) {
+      const weight = weights?.[member.id] ?? 1;
+      totalWeight += weight;
+      weightedSum += member.result.score * weight;
+      allHits.push(...member.result.hits.map((h) => `[${member.id}] ${h}`));
+      allMisses.push(...member.result.misses.map((m) => `[${member.id}] ${m}`));
+      if (member.result.reasoning) {
+        reasoningParts.push(`${member.id}: ${member.result.reasoning}`);
+      }
+      evaluatorResults.push({
+        name: member.id,
+        type: member.type,
+        score: member.result.score,
+        weight,
+        verdict: member.result.verdict,
+        hits: [...member.result.hits],
+        misses: [...member.result.misses],
+        reasoning: member.result.reasoning,
+        evaluatorRawRequest: member.result.evaluatorRawRequest,
+        evaluatorResults: member.result.evaluatorResults
+      });
+    }
+    const finalScore = totalWeight > 0 ? weightedSum / totalWeight : 0;
+    return {
+      score: clampScore(finalScore),
+      verdict: scoreToVerdict(finalScore),
+      hits: allHits,
+      misses: allMisses,
+      expectedAspectCount: Math.max(allHits.length + allMisses.length, 1),
+      reasoning: reasoningParts.length > 0 ? reasoningParts.join("; ") : void 0,
+      evaluatorRawRequest: {
+        aggregator: "weighted_average",
+        ...weights ? { weights } : {}
+      },
+      evaluatorResults
+    };
+  }
+  async runCodeAggregator(results, scriptPath, cwd, weights) {
+    const resultsObject = Object.fromEntries(results.map((r) => [r.id, r.result]));
+    const inputPayload = JSON.stringify({ results: resultsObject }, null, 2);
+    const evaluatorResults = results.map((member) => ({
+      name: member.id,
+      type: member.type,
+      score: member.result.score,
+      weight: weights?.[member.id] ?? 1,
+      verdict: member.result.verdict,
+      hits: [...member.result.hits],
+      misses: [...member.result.misses],
+      reasoning: member.result.reasoning,
+      evaluatorRawRequest: member.result.evaluatorRawRequest,
+      evaluatorResults: member.result.evaluatorResults
+    }));
+    try {
+      const stdout = await executeScript(scriptPath, inputPayload, void 0, cwd);
+      const parsed = parseJsonSafe(stdout);
+      const score = clampScore(typeof parsed?.score === "number" ? parsed.score : 0);
+      const hits = Array.isArray(parsed?.hits) ? parsed.hits.filter(isNonEmptyString) : [];
+      const misses = Array.isArray(parsed?.misses) ? parsed.misses.filter(isNonEmptyString) : [];
+      const reasoning = typeof parsed?.reasoning === "string" ? parsed.reasoning : void 0;
+      const verdict = typeof parsed?.verdict === "string" && (parsed.verdict === "pass" || parsed.verdict === "fail" || parsed.verdict === "borderline") ? parsed.verdict : scoreToVerdict(score);
+      return {
+        score,
+        verdict,
+        hits,
+        misses,
+        expectedAspectCount: hits.length + misses.length || 1,
+        reasoning,
+        evaluatorRawRequest: {
+          aggregator: "code_judge",
+          script: scriptPath
+        },
+        evaluatorResults
+      };
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      return {
+        score: 0,
+        verdict: "fail",
+        hits: [],
+        misses: [`Code aggregator failed: ${message}`],
+        expectedAspectCount: 1,
+        reasoning: message,
+        evaluatorRawRequest: {
+          aggregator: "code_judge",
+          script: scriptPath,
+          error: message
+        },
+        evaluatorResults
+      };
+    }
+  }
+  async runLlmAggregator(results, context, config) {
+    const judgeProvider = context.judgeProvider;
+    if (!judgeProvider) {
+      throw new Error("No judge provider available for LLM aggregation");
+    }
+    const resultsObject = Object.fromEntries(results.map((r) => [r.id, r.result]));
+    const resultsJson = JSON.stringify(resultsObject, null, 2);
+    const evaluatorResults = results.map((member) => ({
+      name: member.id,
+      type: member.type,
+      score: member.result.score,
+      verdict: member.result.verdict,
+      hits: [...member.result.hits],
+      misses: [...member.result.misses],
+      reasoning: member.result.reasoning,
+      evaluatorRawRequest: member.result.evaluatorRawRequest,
+      evaluatorResults: member.result.evaluatorResults
+    }));
+    const promptTemplate = config.prompt ?? DEFAULT_COMPOSITE_AGGREGATOR_PROMPT;
+    const userPrompt = promptTemplate.replace(/\{\{EVALUATOR_RESULTS_JSON\}\}/g, resultsJson);
+    const systemPrompt = buildOutputSchema();
+    const evaluatorRawRequest = {
+      aggregator: "llm_judge",
+      userPrompt,
+      systemPrompt,
+      target: judgeProvider.targetName
+    };
+    try {
+      const model = judgeProvider.asLanguageModel?.();
+      if (model) {
+        const { text } = await (0, import_ai2.generateText)({
+          model,
+          system: systemPrompt,
+          prompt: userPrompt
+        });
+        const data2 = freeformEvaluationSchema.parse(parseJsonFromText(text));
+        const score2 = clampScore(data2.score);
+        const hits2 = Array.isArray(data2.hits) ? data2.hits.filter(isNonEmptyString).slice(0, 4) : [];
+        const misses2 = Array.isArray(data2.misses) ? data2.misses.filter(isNonEmptyString).slice(0, 4) : [];
+        const reasoning2 = data2.reasoning;
+        return {
+          score: score2,
+          verdict: scoreToVerdict(score2),
+          hits: hits2,
+          misses: misses2,
+          expectedAspectCount: Math.max(hits2.length + misses2.length, 1),
+          reasoning: reasoning2,
+          evaluatorRawRequest,
+          evaluatorResults
+        };
+      }
+      const response = await judgeProvider.invoke({
+        question: userPrompt,
+        systemPrompt,
+        evalCaseId: context.evalCase.id,
+        attempt: context.attempt
+      });
+      const data = freeformEvaluationSchema.parse(parseJsonFromText(response.text ?? ""));
+      const score = clampScore(data.score);
+      const hits = Array.isArray(data.hits) ? data.hits.filter(isNonEmptyString).slice(0, 4) : [];
+      const misses = Array.isArray(data.misses) ? data.misses.filter(isNonEmptyString).slice(0, 4) : [];
+      const reasoning = data.reasoning ?? response.reasoning;
+      return {
+        score,
+        verdict: scoreToVerdict(score),
+        hits,
+        misses,
+        expectedAspectCount: Math.max(hits.length + misses.length, 1),
+        reasoning,
+        evaluatorRawRequest,
+        evaluatorResults
+      };
+    } catch {
+      return {
+        score: 0,
+        verdict: "fail",
+        hits: [],
+        misses: [],
+        expectedAspectCount: 1,
+        evaluatorRawRequest,
+        evaluatorResults
+      };
+    }
+  }
+};
 
 // src/evaluation/orchestrator.ts
 var import_node_crypto2 = require("crypto");
@@ -4743,7 +5038,6 @@ async function runEvaluatorList(options) {
           reasoning: score2.reasoning,
           evaluator_provider_request: score2.evaluatorRawRequest
         });
-        continue;
       }
       if (evaluator.type === "code") {
         const codeEvaluator = new CodeEvaluator({
@@ -4760,10 +5054,10 @@ async function runEvaluatorList(options) {
           promptInputs,
           now
         });
-        scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
+        scored.push({ score: score2, name: evaluator.name, type: "code_judge" });
         evaluatorResults.push({
           name: evaluator.name,
-          type: evaluator.type,
+          type: "code_judge",
           score: score2.score,
           verdict: score2.verdict,
           hits: score2.hits,
@@ -4771,19 +5065,37 @@ async function runEvaluatorList(options) {
           reasoning: score2.reasoning,
           evaluator_provider_request: score2.evaluatorRawRequest
         });
-        continue;
       }
-      if (evaluator.type === "rubric") {
-        const rubricEvaluator = new RubricEvaluator({
-          config: evaluator,
-          resolveJudgeProvider: async (context) => {
-            if (context.judgeProvider) {
-              return context.judgeProvider;
+      if (evaluator.type === "composite") {
+        const evalFileDir = evalCase.guideline_paths[0] ? import_node_path13.default.dirname(evalCase.guideline_paths[0]) : process.cwd();
+        const createEvaluator = (memberConfig) => {
+          switch (memberConfig.type) {
+            case "llm_judge":
+              return evaluatorRegistry.llm_judge;
+            case "code":
+              return new CodeEvaluator({
+                script: memberConfig.script,
+                cwd: memberConfig.resolvedCwd ?? memberConfig.cwd,
+                agentTimeoutMs
+              });
+            case "composite":
+              return new CompositeEvaluator({
+                config: memberConfig,
+                cwd: evalFileDir,
+                evaluatorFactory: { create: createEvaluator }
+              });
+            default: {
+              const unknownConfig = memberConfig;
+              throw new Error(`Unsupported evaluator type in composite: ${unknownConfig.type}`);
             }
-            return judgeProvider;
           }
+        };
+        const compositeEvaluator = new CompositeEvaluator({
+          config: evaluator,
+          cwd: evalFileDir,
+          evaluatorFactory: { create: createEvaluator }
         });
-        const score2 = await rubricEvaluator.evaluate({
+        const score2 = await compositeEvaluator.evaluate({
           evalCase,
           candidate,
           target,
@@ -4802,27 +5114,31 @@ async function runEvaluatorList(options) {
           hits: score2.hits,
           misses: score2.misses,
           reasoning: score2.reasoning,
-          evaluator_provider_request: score2.evaluatorRawRequest
+          evaluator_provider_request: score2.evaluatorRawRequest,
+          evaluator_results: mapChildResults(score2.evaluatorResults)
         });
       }
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       const fallbackScore = {
         score: 0,
+        verdict: "fail",
         hits: [],
         misses: [`Evaluator '${evaluator.name}' failed: ${message}`],
         expectedAspectCount: 1,
         reasoning: message
       };
+      const resultType = evaluator.type === "code" ? "code_judge" : evaluator.type;
       scored.push({
         score: fallbackScore,
         name: evaluator.name ?? "unknown",
-        type: evaluator.type ?? "unknown"
+        type: resultType ?? "llm_judge"
       });
       evaluatorResults.push({
         name: evaluator.name ?? "unknown",
-        type: evaluator.type ?? "unknown",
+        type: resultType ?? "llm_judge",
         score: 0,
+        verdict: "fail",
         hits: [],
         misses: [`Evaluator '${evaluator.name ?? "unknown"}' failed: ${message}`],
         reasoning: message
@@ -4841,6 +5157,7 @@ async function runEvaluatorList(options) {
   const reasoning = reasoningParts.length > 0 ? reasoningParts.join(" | ") : void 0;
   const score = {
     score: aggregateScore,
+    verdict: scoreToVerdict2(aggregateScore),
     hits,
     misses,
     expectedAspectCount,
@@ -4891,6 +5208,15 @@ async function resolveCustomPrompt(config) {
 function isNonEmptyString2(value) {
   return typeof value === "string" && value.trim().length > 0;
 }
+function scoreToVerdict2(score) {
+  if (score >= 0.8) {
+    return "pass";
+  }
+  if (score >= 0.6) {
+    return "borderline";
+  }
+  return "fail";
+}
 function filterEvalCases(evalCases, evalId) {
   if (!evalId) {
     return evalCases;
@@ -5028,6 +5354,23 @@ function isTimeoutLike(error) {
   const value = String(error).toLowerCase();
   return value.includes("timeout");
 }
+function mapChildResults(children) {
+  if (!children || children.length === 0) {
+    return void 0;
+  }
+  return children.map((child) => ({
+    name: child.name,
+    type: child.type,
+    score: child.score,
+    weight: child.weight,
+    verdict: child.verdict,
+    hits: child.hits,
+    misses: child.misses,
+    reasoning: child.reasoning,
+    evaluator_provider_request: child.evaluatorRawRequest,
+    evaluator_results: mapChildResults(child.evaluatorResults)
+  }));
+}
 
 // src/evaluation/generators/rubric-generator.ts
 var import_ai3 = require("ai");
@@ -5116,8 +5459,8 @@ function createAgentKernel() {
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   CodeEvaluator,
+  CompositeEvaluator,
   LlmJudgeEvaluator,
-  RubricEvaluator,
   TEST_MESSAGE_ROLES,
   buildDirectoryChain,
   buildPromptInputs,