@agentv/core 3.13.0 → 3.13.2

package/dist/evaluation/validation/index.js

@@ -8,7 +8,7 @@ import {
   isEvaluatorKind,
   loadCasesFromFile,
   resolveFileReference
-} from "../../chunk-4XWPXNQM.js";
+} from "../../chunk-ZB3AUPES.js";
 
 // src/evaluation/validation/file-type.ts
 import { readFile } from "node:fs/promises";

package/dist/index.cjs

@@ -1455,6 +1455,7 @@ __export(index_exports, {
   extractTargetFromSuite: () => extractTargetFromSuite,
   extractTargetsFromSuite: () => extractTargetsFromSuite,
   extractTargetsFromTestCase: () => extractTargetsFromTestCase,
+  extractThreshold: () => extractThreshold,
   extractTrialsConfig: () => extractTrialsConfig,
   extractWorkersFromSuite: () => extractWorkersFromSuite,
   fileExists: () => fileExists2,
@@ -1581,8 +1582,6 @@ function isTestMessage(value) {
 var EVALUATOR_KIND_VALUES = [
   "code-grader",
   "llm-grader",
-  "code-judge",
-  "llm-judge",
   "rubric",
   "composite",
   "tool-trajectory",
@@ -2322,6 +2321,22 @@ function extractFailOnError(suite) {
   logWarning(`Invalid execution.fail_on_error: ${raw}. Must be true or false. Ignoring.`);
   return void 0;
 }
+function extractThreshold(suite) {
+  const execution = suite.execution;
+  if (!execution || typeof execution !== "object" || Array.isArray(execution)) {
+    return void 0;
+  }
+  const executionObj = execution;
+  const raw = executionObj.threshold;
+  if (raw === void 0 || raw === null) {
+    return void 0;
+  }
+  if (typeof raw === "number" && raw >= 0 && raw <= 1) {
+    return raw;
+  }
+  logWarning(`Invalid execution.threshold: ${raw}. Must be a number between 0 and 1. Ignoring.`);
+  return void 0;
+}
 function parseExecutionDefaults(raw, configPath) {
   if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
     return void 0;
@@ -2449,6 +2464,9 @@ var ANSI_RESET5 = "\x1B[0m";
 function normalizeEvaluatorType(type) {
   return type.replace(/_/g, "-");
 }
+function isDeprecatedJudgeType(type) {
+  return type === "code-judge" || type === "llm-judge";
+}
 async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId) {
   const execution = rawEvalCase.execution;
   const executionObject = isJsonObject2(execution) ? execution : void 0;
@@ -2511,6 +2529,12 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
     const rawName = asString(rawEvaluator.name);
     const rawType = rawEvaluator.type;
     const typeValue = typeof rawType === "string" ? normalizeEvaluatorType(rawType) : rawType;
+    if (typeof typeValue === "string" && isDeprecatedJudgeType(typeValue)) {
+      logWarning2(
+        `Skipping evaluator '${rawName ?? "<unnamed>"}' in '${evalId}': '${rawType}' is deprecated. Use '${typeValue.replace("-judge", "-grader")}' instead`
+      );
+      continue;
+    }
     const isCustomType = typeof typeValue === "string" && !isEvaluatorKind(typeValue);
     if (typeof typeValue !== "string") {
       logWarning2(`Skipping evaluator with invalid type in '${evalId}'`);
@@ -2543,7 +2567,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "code-grader" || typeValue === "code-judge") {
+    if (typeValue === "code-grader") {
       let command;
       if (rawEvaluator.script !== void 0 && rawEvaluator.command === void 0) {
         console.warn(
@@ -2653,7 +2677,14 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
         continue;
       }
       const aggregatorType = asString(rawAggregator.type);
-      if (aggregatorType !== "weighted_average" && aggregatorType !== "code-grader" && aggregatorType !== "code-judge" && aggregatorType !== "llm-grader" && aggregatorType !== "llm-judge" && aggregatorType !== "threshold") {
+      const normalizedAggregatorType = typeof aggregatorType === "string" ? aggregatorType === "weighted_average" || aggregatorType === "threshold" ? aggregatorType : normalizeEvaluatorType(aggregatorType) : aggregatorType;
+      if (typeof normalizedAggregatorType === "string" && isDeprecatedJudgeType(normalizedAggregatorType)) {
+        logWarning2(
+          `Skipping composite evaluator '${name}' in '${evalId}': aggregator type '${aggregatorType}' is deprecated. Use '${normalizedAggregatorType.replace("-judge", "-grader")}' instead`
+        );
+        continue;
+      }
+      if (normalizedAggregatorType !== "weighted_average" && normalizedAggregatorType !== "code-grader" && normalizedAggregatorType !== "llm-grader" && normalizedAggregatorType !== "threshold") {
         logWarning2(
           `Skipping composite evaluator '${name}' in '${evalId}': invalid aggregator type '${aggregatorType}'`
         );
@@ -2688,7 +2719,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
         continue;
       }
       let aggregator;
-      if (aggregatorType === "weighted_average") {
+      if (normalizedAggregatorType === "weighted_average") {
         const weights = isJsonObject2(rawAggregator.weights) ? rawAggregator.weights : void 0;
         const parsedWeights = {};
         if (weights) {
@@ -2702,7 +2733,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
           type: "weighted_average",
           ...Object.keys(parsedWeights).length > 0 ? { weights: parsedWeights } : {}
         };
-      } else if (aggregatorType === "code-grader" || aggregatorType === "code-judge") {
+      } else if (normalizedAggregatorType === "code-grader") {
         const aggregatorPath = asString(rawAggregator.path);
         if (!aggregatorPath) {
           logWarning2(
@@ -2715,7 +2746,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
           path: aggregatorPath,
           cwd: searchRoots[0]
         };
-      } else if (aggregatorType === "threshold") {
+      } else if (normalizedAggregatorType === "threshold") {
         const thresholdValue = rawAggregator.threshold;
         if (typeof thresholdValue !== "number" || thresholdValue < 0 || thresholdValue > 1) {
           logWarning2(
@@ -3463,10 +3494,15 @@ function coerceEvaluator(candidate, contextId) {
     return void 0;
   }
   const normalized = normalizeEvaluatorType(candidate);
+  if (isDeprecatedJudgeType(normalized)) {
+    throw new Error(
+      `Unsupported grader '${candidate}' in ${contextId}. Use '${normalized.replace("-judge", "-grader")}' instead.`
+    );
+  }
   if (isEvaluatorKind(normalized)) {
     return normalized;
   }
-  logWarning2(`Unknown evaluator '${candidate}' in ${contextId}, falling back to default`);
+  logWarning2(`Unknown grader '${candidate}' in ${contextId}, falling back to default`);
   return void 0;
 }
 function asString(value) {
@@ -4450,6 +4486,7 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
   const { tests, parsed } = await loadTestsFromYaml(evalFilePath, repoRoot, options);
   const metadata = parseMetadata(parsed);
   const failOnError = extractFailOnError(parsed);
+  const threshold = extractThreshold(parsed);
   return {
     tests,
     trials: extractTrialsConfig(parsed),
@@ -4458,7 +4495,8 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
     cacheConfig: extractCacheConfig(parsed),
     totalBudgetUsd: extractTotalBudgetUsd(parsed),
     ...metadata !== void 0 && { metadata },
-    ...failOnError !== void 0 && { failOnError }
+    ...failOnError !== void 0 && { failOnError },
+    ...threshold !== void 0 && { threshold }
   };
 }
 var loadEvalSuite = loadTestSuite;
@@ -4899,9 +4937,7 @@ function assertionToNaturalLanguage(entry) {
     case "ends_with":
       return `Output ends with '${entry.value}'`;
     case "llm-grader":
-    case "llm_grader":
-    case "llm-judge":
-    case "llm_judge": {
+    case "llm_grader": {
       if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
         return null;
       }
@@ -4914,9 +4950,7 @@ function assertionToNaturalLanguage(entry) {
       return tools ? `Agent called tools in order: ${tools}` : "Agent followed expected tool trajectory";
     }
     case "code-grader":
-    case "code_grader":
-    case "code-judge":
-    case "code_judge": {
+    case "code_grader": {
       const graderName = entry.name ?? deriveGraderNameFromCommand(entry.command) ?? "code-grader";
       const desc = typeof entry.description === "string" ? entry.description : void 0;
       return codeGraderInstruction(graderName, desc);
@@ -4947,7 +4981,7 @@ function assertionToNaturalLanguage(entry) {
   }
 }
 function assertionToNaturalLanguageList(entry) {
-  if (entry.type === "llm-grader" || entry.type === "llm_grader" || entry.type === "llm-judge" || entry.type === "llm_judge") {
+  if (entry.type === "llm-grader" || entry.type === "llm_grader") {
     if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
       return entry.rubrics.map((r) => r.outcome ?? r.criteria ?? r.id).filter((s) => typeof s === "string");
     }
@@ -13168,7 +13202,7 @@ function toCamelCaseDeep(obj) {
 // src/evaluation/evaluators/code-evaluator.ts
 var FILE_BACKED_OUTPUT_THRESHOLD = 5e4;
 var CodeEvaluator = class {
-  kind = "code-judge";
+  kind = "code-grader";
   command;
   cwd;
   agentTimeoutMs;
@@ -13187,7 +13221,7 @@ var CodeEvaluator = class {
     if (outputForPayload) {
       const serialized = JSON.stringify(outputForPayload);
       if (serialized.length > FILE_BACKED_OUTPUT_THRESHOLD) {
-        const tmpDir = await (0, import_promises26.mkdtemp)((0, import_node_path36.join)((0, import_node_os5.tmpdir)(), "agentv-judge-"));
+        const tmpDir = await (0, import_promises26.mkdtemp)((0, import_node_path36.join)((0, import_node_os5.tmpdir)(), "agentv-grader-"));
         outputPath = (0, import_node_path36.join)(tmpDir, "output.json");
         await (0, import_promises26.writeFile)(outputPath, serialized);
         outputForPayload = null;
@@ -13477,7 +13511,7 @@ var LlmGraderEvaluator = class {
       return this.evaluateWithDelegatedAgent(context2, graderProvider);
     }
     const config = context2.evaluator;
-    if ((config?.type === "llm-grader" || config?.type === "llm-judge") && config.rubrics && config.rubrics.length > 0) {
+    if (config?.type === "llm-grader" && config.rubrics && config.rubrics.length > 0) {
       return this.evaluateWithRubrics(context2, graderProvider, config.rubrics);
     }
     return this.evaluateFreeform(context2, graderProvider);
@@ -13662,7 +13696,7 @@ ${context2.fileChanges}`;
     const systemPrompt = this.buildAgentSystemPrompt(context2);
     const userPrompt = this.buildAgentUserPrompt(context2);
     const config = context2.evaluator;
-    const rubrics = config?.type === "llm-grader" || config?.type === "llm-judge" ? config.rubrics : void 0;
+    const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
     const fsTools = createFilesystemTools(workspacePath);
     const evaluatorRawRequest = {
       mode: "built-in",
@@ -13758,7 +13792,7 @@ ${context2.fileChanges}`;
         };
       }
       const config = context2.evaluator;
-      const rubrics = config?.type === "llm-grader" || config?.type === "llm-judge" ? config.rubrics : void 0;
+      const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
       const details = {
         mode: modeLabel,
         grader_target: provider.targetName
@@ -13798,7 +13832,7 @@ ${context2.fileChanges}`;
    */
   buildAgentSystemPrompt(context2) {
     const config = context2.evaluator;
-    const rubrics = config?.type === "llm-grader" || config?.type === "llm-judge" ? config.rubrics : void 0;
+    const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
     const parts = [
       "You are an expert evaluator with access to the workspace filesystem.",
       "Use the provided tools to investigate the workspace and verify the criteria are met.",
@@ -13829,7 +13863,7 @@ ${context2.fileChanges}`;
       return substituteVariables(this.evaluatorTemplate, variables);
     }
     const config = context2.evaluator;
-    const rubrics = config?.type === "llm-grader" || config?.type === "llm-judge" ? config.rubrics : void 0;
+    const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
     const parts = [
       "Evaluate the candidate answer by investigating the workspace.",
       "",
@@ -13872,7 +13906,7 @@ ${context2.fileChanges}`;
   buildDelegatedPrompt(context2) {
     const formattedQuestion = context2.promptInputs.question && context2.promptInputs.question.trim().length > 0 ? context2.promptInputs.question : context2.evalCase.question;
     const config = context2.evaluator;
-    const rubrics = config?.type === "llm-grader" || config?.type === "llm-judge" ? config.rubrics : void 0;
+    const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
     if (this.evaluatorTemplate) {
       const variables = {
         [TEMPLATE_VARIABLES.CRITERIA]: context2.evalCase.criteria.trim(),
@@ -14369,10 +14403,8 @@ var CompositeEvaluator = class {
     const aggregator = this.config.aggregator;
     switch (aggregator.type) {
       case "code-grader":
-      case "code-judge":
         return this.runCodeAggregator(results, aggregator.path, aggregator.cwd ?? this.cwd);
       case "llm-grader":
-      case "llm-judge":
         return this.runLlmAggregator(results, context2, aggregator);
       case "threshold":
         return this.runThreshold(results, aggregator.threshold);
@@ -16794,7 +16826,7 @@ var endsWithFactory = (config) => {
 };
 function createBuiltinRegistry() {
   const registry = new EvaluatorRegistry();
-  registry.register("llm-grader", llmGraderFactory).register("llm-judge", llmGraderFactory).register("code-grader", codeFactory).register("code-judge", codeFactory).register("composite", compositeFactory).register("tool-trajectory", toolTrajectoryFactory).register("field-accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token-usage", tokenUsageFactory).register("execution-metrics", executionMetricsFactory).register("skill-trigger", skillTriggerFactory).register("contains", containsFactory).register("contains-any", containsAnyFactory).register("contains-all", containsAllFactory).register("icontains", icontainsFactory).register("icontains-any", icontainsAnyFactory).register("icontains-all", icontainsAllFactory).register("starts-with", startsWithFactory).register("ends-with", endsWithFactory).register("regex", regexFactory).register("is-json", isJsonFactory).register("equals", equalsFactory).register("inline-assert", (config) => {
+  registry.register("llm-grader", llmGraderFactory).register("code-grader", codeFactory).register("composite", compositeFactory).register("tool-trajectory", toolTrajectoryFactory).register("field-accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token-usage", tokenUsageFactory).register("execution-metrics", executionMetricsFactory).register("skill-trigger", skillTriggerFactory).register("contains", containsFactory).register("contains-any", containsAnyFactory).register("contains-all", containsAllFactory).register("icontains", icontainsFactory).register("icontains-any", icontainsAnyFactory).register("icontains-all", icontainsAllFactory).register("starts-with", startsWithFactory).register("ends-with", endsWithFactory).register("regex", regexFactory).register("is-json", isJsonFactory).register("equals", equalsFactory).register("inline-assert", (config) => {
     const fn = config[INLINE_ASSERT_FN];
     if (!fn) {
       throw new Error(
@@ -19512,7 +19544,7 @@ function filterEvalCases(evalCases, filter) {
   return evalCases.filter((evalCase) => import_micromatch3.default.isMatch(evalCase.id, filter));
 }
 function buildEvaluatorRegistry(overrides, resolveGraderProvider) {
-  const llmGrader = overrides?.["llm-grader"] ?? overrides?.["llm-judge"] ?? new LlmGraderEvaluator({
+  const llmGrader = overrides?.["llm-grader"] ?? new LlmGraderEvaluator({
     resolveGraderProvider: async (context2) => {
       if (context2.graderProvider) {
         return context2.graderProvider;
@@ -20356,10 +20388,10 @@ var OtelTraceExporter = class {
         }
         if (result.scores) {
           for (const score of result.scores) {
-            rootSpan.addEvent(`agentv.evaluator.${score.name}`, {
-              "agentv.evaluator.score": score.score,
-              "agentv.evaluator.type": score.type,
-              ...score.verdict ? { "agentv.evaluator.verdict": score.verdict } : {}
+            rootSpan.addEvent(`agentv.grader.${score.name}`, {
+              "agentv.grader.score": score.score,
+              "agentv.grader.type": score.type,
+              ...score.verdict ? { "agentv.grader.verdict": score.verdict } : {}
             });
           }
         }
@@ -20749,6 +20781,7 @@ function createAgentKernel() {
   extractTargetFromSuite,
   extractTargetsFromSuite,
   extractTargetsFromTestCase,
+  extractThreshold,
   extractTrialsConfig,
   extractWorkersFromSuite,
   fileExists,