@agentv/core 0.22.2 → 0.23.0

package/dist/index.js

@@ -51,7 +51,7 @@ function isTestMessage(value) {
   }
   return candidate.content.every(isJsonObject);
 }
-var EVALUATOR_KIND_VALUES = ["code", "llm_judge", "rubric"];
+var EVALUATOR_KIND_VALUES = ["code_judge", "llm_judge", "rubric", "composite"];
 var EVALUATOR_KIND_SET = new Set(EVALUATOR_KIND_VALUES);
 function isEvaluatorKind(value) {
   return typeof value === "string" && EVALUATOR_KIND_SET.has(value);
@@ -403,10 +403,10 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
       logWarning2(`Skipping evaluator with invalid name/type in '${evalId}'`);
       continue;
     }
-    if (typeValue === "code") {
+    if (typeValue === "code_judge") {
       const script = asString2(rawEvaluator.script);
       if (!script) {
-        logWarning2(`Skipping code evaluator '${name}' in '${evalId}': missing script`);
+        logWarning2(`Skipping code_judge evaluator '${name}' in '${evalId}': missing script`);
         continue;
       }
       const cwd = asString2(rawEvaluator.cwd);
@@ -417,7 +417,7 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
           resolvedCwd = path3.resolve(resolved.resolvedPath);
         } else {
           logWarning2(
-            `Code evaluator '${name}' in '${evalId}': cwd not found (${resolved.displayPath})`,
+            `Code_judge evaluator '${name}' in '${evalId}': cwd not found (${resolved.displayPath})`,
             resolved.attempted.length > 0 ? resolved.attempted.map((attempt) => `  Tried: ${attempt}`) : void 0
           );
         }
@@ -433,6 +433,105 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
       });
       continue;
     }
+    if (typeValue === "composite") {
+      const rawMembers = rawEvaluator.evaluators;
+      if (!Array.isArray(rawMembers)) {
+        logWarning2(
+          `Skipping composite evaluator '${name}' in '${evalId}': missing evaluators array`
+        );
+        continue;
+      }
+      const rawAggregator = rawEvaluator.aggregator;
+      if (!isJsonObject2(rawAggregator)) {
+        logWarning2(`Skipping composite evaluator '${name}' in '${evalId}': missing aggregator`);
+        continue;
+      }
+      const aggregatorType = asString2(rawAggregator.type);
+      if (aggregatorType !== "weighted_average" && aggregatorType !== "code_judge" && aggregatorType !== "llm_judge") {
+        logWarning2(
+          `Skipping composite evaluator '${name}' in '${evalId}': invalid aggregator type '${aggregatorType}'`
+        );
+        continue;
+      }
+      const memberEvaluators = [];
+      for (const rawMember of rawMembers) {
+        if (!isJsonObject2(rawMember)) {
+          logWarning2(`Skipping invalid member evaluator in composite '${name}' (expected object)`);
+          continue;
+        }
+        const memberName = asString2(rawMember.name);
+        const memberType = rawMember.type;
+        if (!memberName || !isEvaluatorKind(memberType)) {
+          logWarning2(`Skipping member evaluator with invalid name/type in composite '${name}'`);
+          continue;
+        }
+        const memberConfigs = await parseEvaluators(
+          { evaluators: [rawMember] },
+          void 0,
+          searchRoots,
+          `${evalId}:${name}:${memberName}`
+        );
+        if (memberConfigs && memberConfigs.length > 0) {
+          memberEvaluators.push(memberConfigs[0]);
+        }
+      }
+      if (memberEvaluators.length === 0) {
+        logWarning2(
+          `Skipping composite evaluator '${name}' in '${evalId}': no valid member evaluators`
+        );
+        continue;
+      }
+      let aggregator;
+      if (aggregatorType === "weighted_average") {
+        const weights = isJsonObject2(rawAggregator.weights) ? rawAggregator.weights : void 0;
+        const parsedWeights = {};
+        if (weights) {
+          for (const [key, value] of Object.entries(weights)) {
+            if (typeof value === "number") {
+              parsedWeights[key] = value;
+            }
+          }
+        }
+        aggregator = {
+          type: "weighted_average",
+          ...Object.keys(parsedWeights).length > 0 ? { weights: parsedWeights } : {}
+        };
+      } else if (aggregatorType === "code_judge") {
+        const aggregatorPath = asString2(rawAggregator.path);
+        if (!aggregatorPath) {
+          logWarning2(
+            `Skipping composite evaluator '${name}' in '${evalId}': code_judge aggregator missing path`
+          );
+          continue;
+        }
+        aggregator = {
+          type: "code_judge",
+          path: aggregatorPath,
+          cwd: searchRoots[0]
+        };
+      } else {
+        const aggregatorPrompt = asString2(rawAggregator.prompt);
+        let promptPath2;
+        if (aggregatorPrompt) {
+          const resolved = await resolveFileReference2(aggregatorPrompt, searchRoots);
+          if (resolved.resolvedPath) {
+            promptPath2 = path3.resolve(resolved.resolvedPath);
+          }
+        }
+        aggregator = {
+          type: "llm_judge",
+          ...aggregatorPrompt ? { prompt: aggregatorPrompt } : {},
+          ...promptPath2 ? { promptPath: promptPath2 } : {}
+        };
+      }
+      evaluators.push({
+        name,
+        type: "composite",
+        evaluators: memberEvaluators,
+        aggregator
+      });
+      continue;
+    }
     const prompt = asString2(rawEvaluator.prompt);
     let promptPath;
     if (prompt) {
@@ -3329,6 +3428,228 @@ function substituteVariables(template, variables) {
     return variables[varName] ?? match;
   });
 }
+var DEFAULT_COMPOSITE_AGGREGATOR_PROMPT = `Review the following evaluation results:
+{{EVALUATOR_RESULTS_JSON}}
+
+Decide the final score and verdict based on all evaluator results.
+Return a JSON object with: score (0.0-1.0), verdict (pass/fail/borderline), and reasoning.`;
+var CompositeEvaluator = class {
+  kind = "composite";
+  config;
+  evaluatorFactory;
+  cwd;
+  constructor(options) {
+    this.config = options.config;
+    this.evaluatorFactory = options.evaluatorFactory;
+    this.cwd = options.cwd;
+  }
+  async evaluate(context) {
+    const memberResults = await Promise.all(
+      this.config.evaluators.map(async (memberConfig) => {
+        const evaluator = this.evaluatorFactory.create(memberConfig, context);
+        return {
+          id: memberConfig.name,
+          type: memberConfig.type,
+          result: await evaluator.evaluate(context)
+        };
+      })
+    );
+    return this.aggregate(memberResults, context);
+  }
+  async aggregate(results, context) {
+    const aggregator = this.config.aggregator;
+    switch (aggregator.type) {
+      case "code_judge":
+        return this.runCodeAggregator(results, aggregator.path, aggregator.cwd ?? this.cwd);
+      case "llm_judge":
+        return this.runLlmAggregator(results, context, aggregator);
+      default:
+        return this.runWeightedAverage(results, aggregator.weights);
+    }
+  }
+  runWeightedAverage(results, weights) {
+    let totalWeight = 0;
+    let weightedSum = 0;
+    const allHits = [];
+    const allMisses = [];
+    const reasoningParts = [];
+    const evaluatorResults = [];
+    for (const member of results) {
+      const weight = weights?.[member.id] ?? 1;
+      totalWeight += weight;
+      weightedSum += member.result.score * weight;
+      allHits.push(...member.result.hits.map((h) => `[${member.id}] ${h}`));
+      allMisses.push(...member.result.misses.map((m) => `[${member.id}] ${m}`));
+      if (member.result.reasoning) {
+        reasoningParts.push(`${member.id}: ${member.result.reasoning}`);
+      }
+      evaluatorResults.push({
+        name: member.id,
+        type: member.type,
+        score: member.result.score,
+        weight,
+        verdict: member.result.verdict,
+        hits: [...member.result.hits],
+        misses: [...member.result.misses],
+        reasoning: member.result.reasoning,
+        evaluatorRawRequest: member.result.evaluatorRawRequest,
+        evaluatorResults: member.result.evaluatorResults
+      });
+    }
+    const finalScore = totalWeight > 0 ? weightedSum / totalWeight : 0;
+    return {
+      score: clampScore(finalScore),
+      verdict: scoreToVerdict(finalScore),
+      hits: allHits,
+      misses: allMisses,
+      expectedAspectCount: Math.max(allHits.length + allMisses.length, 1),
+      reasoning: reasoningParts.length > 0 ? reasoningParts.join("; ") : void 0,
+      evaluatorRawRequest: {
+        aggregator: "weighted_average",
+        ...weights ? { weights } : {}
+      },
+      evaluatorResults
+    };
+  }
+  async runCodeAggregator(results, scriptPath, cwd, weights) {
+    const resultsObject = Object.fromEntries(results.map((r) => [r.id, r.result]));
+    const inputPayload = JSON.stringify({ results: resultsObject }, null, 2);
+    const evaluatorResults = results.map((member) => ({
+      name: member.id,
+      type: member.type,
+      score: member.result.score,
+      weight: weights?.[member.id] ?? 1,
+      verdict: member.result.verdict,
+      hits: [...member.result.hits],
+      misses: [...member.result.misses],
+      reasoning: member.result.reasoning,
+      evaluatorRawRequest: member.result.evaluatorRawRequest,
+      evaluatorResults: member.result.evaluatorResults
+    }));
+    try {
+      const stdout = await executeScript(scriptPath, inputPayload, void 0, cwd);
+      const parsed = parseJsonSafe(stdout);
+      const score = clampScore(typeof parsed?.score === "number" ? parsed.score : 0);
+      const hits = Array.isArray(parsed?.hits) ? parsed.hits.filter(isNonEmptyString) : [];
+      const misses = Array.isArray(parsed?.misses) ? parsed.misses.filter(isNonEmptyString) : [];
+      const reasoning = typeof parsed?.reasoning === "string" ? parsed.reasoning : void 0;
+      const verdict = typeof parsed?.verdict === "string" && (parsed.verdict === "pass" || parsed.verdict === "fail" || parsed.verdict === "borderline") ? parsed.verdict : scoreToVerdict(score);
+      return {
+        score,
+        verdict,
+        hits,
+        misses,
+        expectedAspectCount: hits.length + misses.length || 1,
+        reasoning,
+        evaluatorRawRequest: {
+          aggregator: "code_judge",
+          script: scriptPath
+        },
+        evaluatorResults
+      };
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      return {
+        score: 0,
+        verdict: "fail",
+        hits: [],
+        misses: [`Code aggregator failed: ${message}`],
+        expectedAspectCount: 1,
+        reasoning: message,
+        evaluatorRawRequest: {
+          aggregator: "code_judge",
+          script: scriptPath,
+          error: message
+        },
+        evaluatorResults
+      };
+    }
+  }
+  async runLlmAggregator(results, context, config) {
+    const judgeProvider = context.judgeProvider;
+    if (!judgeProvider) {
+      throw new Error("No judge provider available for LLM aggregation");
+    }
+    const resultsObject = Object.fromEntries(results.map((r) => [r.id, r.result]));
+    const resultsJson = JSON.stringify(resultsObject, null, 2);
+    const evaluatorResults = results.map((member) => ({
+      name: member.id,
+      type: member.type,
+      score: member.result.score,
+      verdict: member.result.verdict,
+      hits: [...member.result.hits],
+      misses: [...member.result.misses],
+      reasoning: member.result.reasoning,
+      evaluatorRawRequest: member.result.evaluatorRawRequest,
+      evaluatorResults: member.result.evaluatorResults
+    }));
+    const promptTemplate = config.prompt ?? DEFAULT_COMPOSITE_AGGREGATOR_PROMPT;
+    const userPrompt = promptTemplate.replace(/\{\{EVALUATOR_RESULTS_JSON\}\}/g, resultsJson);
+    const systemPrompt = buildOutputSchema();
+    const evaluatorRawRequest = {
+      aggregator: "llm_judge",
+      userPrompt,
+      systemPrompt,
+      target: judgeProvider.targetName
+    };
+    try {
+      const model = judgeProvider.asLanguageModel?.();
+      if (model) {
+        const { text } = await generateText2({
+          model,
+          system: systemPrompt,
+          prompt: userPrompt
+        });
+        const data2 = freeformEvaluationSchema.parse(parseJsonFromText(text));
+        const score2 = clampScore(data2.score);
+        const hits2 = Array.isArray(data2.hits) ? data2.hits.filter(isNonEmptyString).slice(0, 4) : [];
+        const misses2 = Array.isArray(data2.misses) ? data2.misses.filter(isNonEmptyString).slice(0, 4) : [];
+        const reasoning2 = data2.reasoning;
+        return {
+          score: score2,
+          verdict: scoreToVerdict(score2),
+          hits: hits2,
+          misses: misses2,
+          expectedAspectCount: Math.max(hits2.length + misses2.length, 1),
+          reasoning: reasoning2,
+          evaluatorRawRequest,
+          evaluatorResults
+        };
+      }
+      const response = await judgeProvider.invoke({
+        question: userPrompt,
+        systemPrompt,
+        evalCaseId: context.evalCase.id,
+        attempt: context.attempt
+      });
+      const data = freeformEvaluationSchema.parse(parseJsonFromText(response.text ?? ""));
+      const score = clampScore(data.score);
+      const hits = Array.isArray(data.hits) ? data.hits.filter(isNonEmptyString).slice(0, 4) : [];
+      const misses = Array.isArray(data.misses) ? data.misses.filter(isNonEmptyString).slice(0, 4) : [];
+      const reasoning = data.reasoning ?? response.reasoning;
+      return {
+        score,
+        verdict: scoreToVerdict(score),
+        hits,
+        misses,
+        expectedAspectCount: Math.max(hits.length + misses.length, 1),
+        reasoning,
+        evaluatorRawRequest,
+        evaluatorResults
+      };
+    } catch {
+      return {
+        score: 0,
+        verdict: "fail",
+        hits: [],
+        misses: [],
+        expectedAspectCount: 1,
+        evaluatorRawRequest,
+        evaluatorResults
+      };
+    }
+  }
+};
 
 // src/evaluation/orchestrator.ts
 import { createHash, randomUUID as randomUUID2 } from "node:crypto";
@@ -4030,6 +4351,57 @@ async function runEvaluatorList(options) {
           promptInputs,
           now
         });
+        scored.push({ score: score2, name: evaluator.name, type: "code_judge" });
+        evaluatorResults.push({
+          name: evaluator.name,
+          type: "code_judge",
+          score: score2.score,
+          verdict: score2.verdict,
+          hits: score2.hits,
+          misses: score2.misses,
+          reasoning: score2.reasoning,
+          evaluator_provider_request: score2.evaluatorRawRequest
+        });
+      }
+      if (evaluator.type === "composite") {
+        const evalFileDir = evalCase.guideline_paths[0] ? path12.dirname(evalCase.guideline_paths[0]) : process.cwd();
+        const createEvaluator = (memberConfig) => {
+          switch (memberConfig.type) {
+            case "llm_judge":
+              return evaluatorRegistry.llm_judge;
+            case "code":
+              return new CodeEvaluator({
+                script: memberConfig.script,
+                cwd: memberConfig.resolvedCwd ?? memberConfig.cwd,
+                agentTimeoutMs
+              });
+            case "composite":
+              return new CompositeEvaluator({
+                config: memberConfig,
+                cwd: evalFileDir,
+                evaluatorFactory: { create: createEvaluator }
+              });
+            default: {
+              const unknownConfig = memberConfig;
+              throw new Error(`Unsupported evaluator type in composite: ${unknownConfig.type}`);
+            }
+          }
+        };
+        const compositeEvaluator = new CompositeEvaluator({
+          config: evaluator,
+          cwd: evalFileDir,
+          evaluatorFactory: { create: createEvaluator }
+        });
+        const score2 = await compositeEvaluator.evaluate({
+          evalCase,
+          candidate,
+          target,
+          provider,
+          attempt,
+          promptInputs,
+          now,
+          judgeProvider
+        });
         scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
         evaluatorResults.push({
           name: evaluator.name,
@@ -4039,7 +4411,8 @@ async function runEvaluatorList(options) {
           hits: score2.hits,
           misses: score2.misses,
           reasoning: score2.reasoning,
-          evaluator_provider_request: score2.evaluatorRawRequest
+          evaluator_provider_request: score2.evaluatorRawRequest,
+          evaluator_results: mapChildResults(score2.evaluatorResults)
         });
       }
     } catch (error) {
@@ -4052,14 +4425,15 @@ async function runEvaluatorList(options) {
         expectedAspectCount: 1,
         reasoning: message
       };
+      const resultType = evaluator.type === "code" ? "code_judge" : evaluator.type;
       scored.push({
         score: fallbackScore,
         name: evaluator.name ?? "unknown",
-        type: evaluator.type ?? "unknown"
+        type: resultType ?? "llm_judge"
       });
       evaluatorResults.push({
         name: evaluator.name ?? "unknown",
-        type: evaluator.type ?? "unknown",
+        type: resultType ?? "llm_judge",
         score: 0,
         verdict: "fail",
         hits: [],
@@ -4277,6 +4651,23 @@ function isTimeoutLike(error) {
   const value = String(error).toLowerCase();
   return value.includes("timeout");
 }
+function mapChildResults(children) {
+  if (!children || children.length === 0) {
+    return void 0;
+  }
+  return children.map((child) => ({
+    name: child.name,
+    type: child.type,
+    score: child.score,
+    weight: child.weight,
+    verdict: child.verdict,
+    hits: child.hits,
+    misses: child.misses,
+    reasoning: child.reasoning,
+    evaluator_provider_request: child.evaluatorRawRequest,
+    evaluator_results: mapChildResults(child.evaluatorResults)
+  }));
+}
 
 // src/evaluation/generators/rubric-generator.ts
 import { generateText as generateText3 } from "ai";
@@ -4364,6 +4755,7 @@ function createAgentKernel() {
 }
 export {
   CodeEvaluator,
+  CompositeEvaluator,
   LlmJudgeEvaluator,
   TEST_MESSAGE_ROLES,
   buildDirectoryChain,