@agentv/core 0.26.0 → 1.2.0

package/dist/index.js

@@ -9,7 +9,7 @@ import {
   readTextFile,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-NDEN3H2B.js";
+} from "./chunk-V3JCB3HI.js";
 
 // src/evaluation/types.ts
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
@@ -47,18 +47,23 @@ function isTestMessage(value) {
   if (typeof candidate.content === "string") {
     return true;
   }
-  if (!Array.isArray(candidate.content)) {
-    return false;
+  if (Array.isArray(candidate.content) && candidate.content.every(isJsonObject)) {
+    return true;
+  }
+  if (Array.isArray(candidate.tool_calls) && candidate.tool_calls.length > 0) {
+    return true;
   }
-  return candidate.content.every(isJsonObject);
+  if (isJsonObject(candidate.content)) {
+    return true;
+  }
+  return false;
 }
 var EVALUATOR_KIND_VALUES = [
   "code_judge",
   "llm_judge",
   "rubric",
   "composite",
-  "tool_trajectory",
-  "expected_messages"
+  "tool_trajectory"
 ];
 var EVALUATOR_KIND_SET = new Set(EVALUATOR_KIND_VALUES);
 function isEvaluatorKind(value) {
@@ -79,13 +84,6 @@ function isTraceEvent(value) {
   const candidate = value;
   return isTraceEventType(candidate.type) && typeof candidate.timestamp === "string";
 }
-function isExpectedToolCall(value) {
-  if (typeof value !== "object" || value === null) {
-    return false;
-  }
-  const candidate = value;
-  return typeof candidate.tool === "string";
-}
 function computeTraceSummary(trace) {
   const toolCallCounts = {};
   let errorCount = 0;
@@ -582,15 +580,6 @@ async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId
       });
       continue;
     }
-    if (typeValue === "expected_messages") {
-      const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
-      evaluators.push({
-        name,
-        type: "expected_messages",
-        ...weight2 !== void 0 ? { weight: weight2 } : {}
-      });
-      continue;
-    }
     if (typeValue === "tool_trajectory") {
       const mode = asString2(rawEvaluator.mode);
       if (mode !== "any_order" && mode !== "in_order" && mode !== "exact") {
@@ -845,63 +834,6 @@ async function processMessages(options) {
   }
   return segments;
 }
-async function resolveAssistantContent(content, searchRoots, verbose) {
-  if (typeof content === "string") {
-    return content;
-  }
-  if (!content) {
-    return "";
-  }
-  const parts = [];
-  for (const entry of content) {
-    if (typeof entry === "string") {
-      parts.push({ content: entry, isFile: false });
-      continue;
-    }
-    if (!isJsonObject(entry)) {
-      continue;
-    }
-    const segmentType = asString3(entry.type);
-    if (segmentType === "file") {
-      const rawValue = asString3(entry.value);
-      if (!rawValue) {
-        continue;
-      }
-      const { displayPath, resolvedPath, attempted } = await resolveFileReference2(
-        rawValue,
-        searchRoots
-      );
-      if (!resolvedPath) {
-        const attempts = attempted.length ? ["  Tried:", ...attempted.map((candidate) => `    ${candidate}`)] : void 0;
-        logWarning3(`File not found in expected_messages: ${displayPath}`, attempts);
-        continue;
-      }
-      try {
-        const fileContent = (await readFile3(resolvedPath, "utf8")).replace(/\r\n/g, "\n").trim();
-        parts.push({ content: fileContent, isFile: true, displayPath });
-        if (verbose) {
-          console.log(`  [Expected Assistant File] Found: ${displayPath}`);
-          console.log(`    Resolved to: ${resolvedPath}`);
-        }
-      } catch (error) {
-        logWarning3(`Could not read file ${resolvedPath}: ${error.message}`);
-      }
-      continue;
-    }
-    const textValue = asString3(entry.text);
-    if (typeof textValue === "string") {
-      parts.push({ content: textValue, isFile: false });
-      continue;
-    }
-    const valueValue = asString3(entry.value);
-    if (typeof valueValue === "string") {
-      parts.push({ content: valueValue, isFile: false });
-      continue;
-    }
-    parts.push({ content: JSON.stringify(entry), isFile: false });
-  }
-  return formatFileContents(parts);
-}
 function asString3(value) {
   return typeof value === "string" ? value : void 0;
 }
@@ -934,14 +866,15 @@ ${detailBlock}${ANSI_RESET4}`);
   }
 }
 async function processExpectedMessages(options) {
-  const { messages, searchRoots, repoRootPath, verbose } = options;
+  const { messages, searchRoots, verbose } = options;
   const segments = [];
   for (const message of messages) {
+    const extendedMessage = message;
     const segment = {
       role: message.role
     };
-    if (message.role === "assistant" && message.tool_calls !== void 0) {
-      segment.tool_calls = message.tool_calls;
+    if (extendedMessage.name) {
+      segment.name = extendedMessage.name;
     }
     const content = message.content;
     if (typeof content === "string") {
@@ -989,6 +922,13 @@ async function processExpectedMessages(options) {
         processedContent.push(cloneJsonObject(rawSegment));
       }
       segment.content = processedContent;
+    } else if (isJsonObject(content)) {
+      segment.content = cloneJsonObject(content);
+    }
+    if (extendedMessage.tool_calls && Array.isArray(extendedMessage.tool_calls)) {
+      segment.tool_calls = extendedMessage.tool_calls.map(
+        (tc) => isJsonObject(tc) ? cloneJsonObject(tc) : tc
+      );
     }
     segments.push(segment);
   }
@@ -1283,9 +1223,6 @@ async function loadEvalCases(evalFilePath, repoRoot, options) {
       logError(`No valid expected message found for eval case: ${id}`);
       continue;
     }
-    if (expectedMessages.length > 1) {
-      logWarning5(`Multiple expected messages found for eval case: ${id}, using first`);
-    }
     const guidelinePaths = [];
     const inputTextParts = [];
     const inputSegments = await processMessages({
@@ -1305,8 +1242,19 @@ async function loadEvalCases(evalFilePath, repoRoot, options) {
       verbose
     }) : [];
     const codeSnippets = extractCodeBlocks(inputSegments);
-    const expectedContent = expectedMessages[0]?.content;
-    const referenceAnswer = expectedContent ? await resolveAssistantContent(expectedContent, searchRoots, verbose) : "";
+    let referenceAnswer = "";
+    if (outputSegments.length > 1) {
+      referenceAnswer = JSON.stringify(outputSegments, null, 2);
+    } else if (outputSegments.length === 1) {
+      const singleMessage = outputSegments[0];
+      if (typeof singleMessage.content === "string") {
+        referenceAnswer = singleMessage.content;
+      } else if (singleMessage.content) {
+        referenceAnswer = JSON.stringify(singleMessage, null, 2);
+      } else if (singleMessage.tool_calls) {
+        referenceAnswer = JSON.stringify(singleMessage, null, 2);
+      }
+    }
     const question = inputTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
     const evalCaseEvaluatorKind = coerceEvaluator(evalcase.evaluator, id) ?? globalEvaluator;
     let evaluators;
@@ -1361,7 +1309,7 @@ async function loadEvalCases(evalFilePath, repoRoot, options) {
       question,
       input_messages: inputMessages,
       input_segments: inputSegments,
-      expected_segments: outputSegments,
+      expected_messages: outputSegments,
       reference_answer: referenceAnswer,
       guideline_paths: guidelinePaths.map((guidelinePath) => path6.resolve(guidelinePath)),
       guideline_patterns: guidelinePatterns,
@@ -3270,7 +3218,7 @@ import { generateText as generateText2 } from "ai";
 import { z } from "zod";
 var DEFAULT_EVALUATOR_TEMPLATE = `You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.
 
-Use the reference_answer as a gold standard for a high-quality response (if provided). The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.
+Use the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.
 
 Be concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.
 
@@ -3328,7 +3276,7 @@ var LlmJudgeEvaluator = class {
     const variables = {
       [TEMPLATE_VARIABLES.INPUT_MESSAGES]: JSON.stringify(context.evalCase.input_segments, null, 2),
       [TEMPLATE_VARIABLES.EXPECTED_MESSAGES]: JSON.stringify(
-        context.evalCase.expected_segments,
+        context.evalCase.expected_messages,
         null,
         2
       ),
@@ -3547,7 +3495,9 @@ var CodeEvaluator = class {
         input_files: context.evalCase.file_paths.filter(
           (path13) => !context.evalCase.guideline_paths.includes(path13)
         ),
-        input_messages: context.evalCase.input_messages
+        input_messages: context.evalCase.input_messages,
+        candidate_trace_file: context.candidateTraceRef ?? null,
+        candidate_trace_summary: context.candidateTraceSummary ?? null
       },
       null,
       2
@@ -3813,105 +3763,6 @@ var ToolTrajectoryEvaluator = class {
     };
   }
 };
-var ExpectedMessagesEvaluator = class {
-  kind = "expected_messages";
-  evaluate(context) {
-    const { candidateTrace, evalCase } = context;
-    const expectedSegments = evalCase.expected_segments;
-    const expectedToolCalls = this.extractExpectedToolCalls(expectedSegments);
-    if (expectedToolCalls.length === 0) {
-      return {
-        score: 1,
-        verdict: "pass",
-        hits: ["No tool_calls specified in expected_messages"],
-        misses: [],
-        expectedAspectCount: 1
-      };
-    }
-    if (!candidateTrace || candidateTrace.length === 0) {
-      return {
-        score: 0,
-        verdict: "fail",
-        hits: [],
-        misses: ["No trace available to validate tool_calls"],
-        expectedAspectCount: expectedToolCalls.length
-      };
-    }
-    const actualToolCalls = candidateTrace.filter((e) => e.type === "tool_call");
-    return this.validateToolCalls(expectedToolCalls, actualToolCalls);
-  }
-  extractExpectedToolCalls(segments) {
-    if (!segments) {
-      return [];
-    }
-    const toolCalls = [];
-    for (const segment of segments) {
-      const role = segment.role;
-      const segmentToolCalls = segment.tool_calls;
-      if (role === "assistant" && Array.isArray(segmentToolCalls)) {
-        for (const tc of segmentToolCalls) {
-          if (typeof tc === "object" && tc !== null && typeof tc.tool === "string") {
-            const toolCall = tc;
-            toolCalls.push({ tool: toolCall.tool, input: toolCall.input });
-          }
-        }
-      }
-    }
-    return toolCalls;
-  }
-  validateToolCalls(expected, actual) {
-    const hits = [];
-    const misses = [];
-    for (let i = 0; i < expected.length; i++) {
-      const expectedCall = expected[i];
-      const actualCall = actual[i];
-      if (!actualCall) {
-        misses.push(
-          `tool_calls[${i}]: expected ${expectedCall.tool}, but no more tool calls in trace`
-        );
-        continue;
-      }
-      if (actualCall.name !== expectedCall.tool) {
-        misses.push(
-          `tool_calls[${i}]: expected ${expectedCall.tool}, got ${actualCall.name ?? "unknown"}`
-        );
-        continue;
-      }
-      if (expectedCall.input !== void 0) {
-        if (!this.deepEquals(expectedCall.input, actualCall.input)) {
-          misses.push(`tool_calls[${i}]: ${expectedCall.tool} input mismatch`);
-          continue;
-        }
-      }
-      hits.push(`tool_calls[${i}]: ${expectedCall.tool} matched`);
-    }
-    const totalChecks = expected.length || 1;
-    const score = hits.length / totalChecks;
-    return {
-      score,
-      verdict: score >= 0.8 ? "pass" : score >= 0.6 ? "borderline" : "fail",
-      hits,
-      misses,
-      expectedAspectCount: totalChecks
-    };
-  }
-  deepEquals(a, b) {
-    if (a === b) return true;
-    if (typeof a !== typeof b) return false;
-    if (typeof a !== "object" || a === null || b === null) return false;
-    if (Array.isArray(a) && Array.isArray(b)) {
-      if (a.length !== b.length) return false;
-      return a.every((val, i) => this.deepEquals(val, b[i]));
-    }
-    if (Array.isArray(a) || Array.isArray(b)) return false;
-    const aObj = a;
-    const bObj = b;
-    const aKeys = Object.keys(aObj);
-    const bKeys = Object.keys(bObj);
-    if (aKeys.length !== bKeys.length) return false;
-    return aKeys.every((key) => this.deepEquals(aObj[key], bObj[key]));
-  }
-};
 var DEFAULT_COMPOSITE_AGGREGATOR_PROMPT = `Review the following evaluation results:
 {{EVALUATOR_RESULTS_JSON}}
 
@@ -4673,6 +4524,7 @@ async function runEvalCase(options) {
       judgeProvider,
       agentTimeoutMs,
       candidateTrace,
+      candidateTraceRef: providerResponse.traceRef,
       candidateTraceSummary
     });
   } catch (error) {
@@ -4692,6 +4544,7 @@ async function evaluateCandidate(options) {
     judgeProvider,
     agentTimeoutMs,
     candidateTrace,
+    candidateTraceRef,
     candidateTraceSummary
   } = options;
   const gradeTimestamp = nowFn();
@@ -4707,6 +4560,7 @@ async function evaluateCandidate(options) {
     judgeProvider,
     agentTimeoutMs,
     candidateTrace,
+    candidateTraceRef,
     candidateTraceSummary
   });
   const completedAt = nowFn();
@@ -4761,6 +4615,7 @@ async function runEvaluatorsForCase(options) {
     judgeProvider,
     agentTimeoutMs,
     candidateTrace,
+    candidateTraceRef,
     candidateTraceSummary
   } = options;
   if (evalCase.evaluators && evalCase.evaluators.length > 0) {
@@ -4777,6 +4632,7 @@ async function runEvaluatorsForCase(options) {
       judgeProvider,
       agentTimeoutMs,
       candidateTrace,
+      candidateTraceRef,
       candidateTraceSummary
     });
   }
@@ -4795,6 +4651,7 @@ async function runEvaluatorsForCase(options) {
     now,
     judgeProvider,
     candidateTrace,
+    candidateTraceRef,
     candidateTraceSummary
   });
   return { score };
@@ -4813,6 +4670,7 @@ async function runEvaluatorList(options) {
     judgeProvider,
     agentTimeoutMs,
     candidateTrace,
+    candidateTraceRef,
     candidateTraceSummary
   } = options;
   const scored = [];
@@ -4859,7 +4717,9 @@ async function runEvaluatorList(options) {
           provider,
           attempt,
           promptInputs,
-          now
+          now,
+          candidateTraceRef,
+          candidateTraceSummary
         });
         const weight = evaluator.weight ?? 1;
         scored.push({ score: score2, name: evaluator.name, type: "code_judge", weight });
@@ -4897,8 +4757,6 @@ async function runEvaluatorList(options) {
               return new ToolTrajectoryEvaluator({
                 config: memberConfig
               });
-            case "expected_messages":
-              return new ExpectedMessagesEvaluator();
             default: {
               const unknownConfig = memberConfig;
               throw new Error(`Unsupported evaluator type in composite: ${unknownConfig.type}`);
@@ -4948,32 +4806,7 @@ async function runEvaluatorList(options) {
           promptInputs,
           now,
           candidateTrace,
-          candidateTraceSummary
-        });
-        const weight = evaluator.weight ?? 1;
-        scored.push({ score: score2, name: evaluator.name, type: evaluator.type, weight });
-        evaluatorResults.push({
-          name: evaluator.name,
-          type: evaluator.type,
-          score: score2.score,
-          weight,
-          verdict: score2.verdict,
-          hits: score2.hits,
-          misses: score2.misses,
-          reasoning: score2.reasoning
-        });
-      }
-      if (evaluator.type === "expected_messages") {
-        const expectedMessagesEvaluator = new ExpectedMessagesEvaluator();
-        const score2 = expectedMessagesEvaluator.evaluate({
-          evalCase,
-          candidate,
-          target,
-          provider,
-          attempt,
-          promptInputs,
-          now,
-          candidateTrace,
+          candidateTraceRef,
           candidateTraceSummary
         });
         const weight = evaluator.weight ?? 1;
@@ -5345,7 +5178,6 @@ function createAgentKernel() {
 export {
   CodeEvaluator,
   CompositeEvaluator,
-  ExpectedMessagesEvaluator,
   LlmJudgeEvaluator,
   TEST_MESSAGE_ROLES,
   ToolTrajectoryEvaluator,
@@ -5363,7 +5195,6 @@ export {
   generateRubrics,
   getHitCount,
   isEvaluatorKind,
-  isExpectedToolCall,
   isGuidelineFile,
   isJsonObject,
   isJsonValue,