agentv 3.13.0 → 3.13.1

package/README.md

@@ -221,13 +221,13 @@ agentv eval evals/my-eval.yaml -o results.xml
 
 The HTML report auto-refreshes every 2 seconds during a live run, then locks once the run completes.
 
-By default, `agentv eval` creates a run workspace under `.agentv/results/raw/<run>/`
+By default, `agentv eval` creates a run workspace under `.agentv/results/runs/<run>/`
 with `index.jsonl` as the machine-facing manifest.
 
 You can also convert an existing manifest to HTML after the fact:
 
 ```bash
-agentv convert .agentv/results/raw/eval_<timestamp>/index.jsonl -o report.html
+agentv convert .agentv/results/runs/eval_<timestamp>/index.jsonl -o report.html
 ```
 
 #### Timeouts
@@ -358,7 +358,7 @@ agentv create eval my-eval          # → evals/my-eval.eval.yaml + .cases.jsonl
 Compare a combined results file across all targets (N-way matrix):
 
 ```bash
-agentv compare .agentv/results/raw/eval_<timestamp>/index.jsonl
+agentv compare .agentv/results/runs/eval_<timestamp>/index.jsonl
 ```
 
 ```
@@ -379,8 +379,8 @@ Pairwise Summary:
 Designate a baseline for CI regression gating, or compare two specific targets:
 
 ```bash
-agentv compare .agentv/results/raw/eval_<timestamp>/index.jsonl --baseline gpt-4.1
-agentv compare .agentv/results/raw/eval_<timestamp>/index.jsonl --baseline gpt-4.1 --candidate gpt-5-mini
+agentv compare .agentv/results/runs/eval_<timestamp>/index.jsonl --baseline gpt-4.1
+agentv compare .agentv/results/runs/eval_<timestamp>/index.jsonl --baseline gpt-4.1 --candidate gpt-5-mini
 agentv compare before.jsonl after.jsonl                                  # two-file pairwise
 ```
 

package/dist/{chunk-7OHZAFND.js → chunk-K747KGDP.js}

@@ -301,7 +301,7 @@ var require_dist = __commonJS({
   }
 });
 
-// ../../packages/core/dist/chunk-4XWPXNQM.js
+// ../../packages/core/dist/chunk-ZB3AUPES.js
 import { constants } from "node:fs";
 import { access, readFile } from "node:fs/promises";
 import path from "node:path";
@@ -419,7 +419,7 @@ __export(external_exports2, {
   void: () => voidType
 });
 
-// ../../packages/core/dist/chunk-4XWPXNQM.js
+// ../../packages/core/dist/chunk-ZB3AUPES.js
 import { readFile as readFile2 } from "node:fs/promises";
 import path3 from "node:path";
 import fg from "fast-glob";
@@ -473,8 +473,6 @@ function isTestMessage(value) {
 var EVALUATOR_KIND_VALUES = [
   "code-grader",
   "llm-grader",
-  "code-judge",
-  "llm-judge",
   "rubric",
   "composite",
   "tool-trajectory",
@@ -14960,6 +14958,9 @@ var ANSI_RESET4 = "\x1B[0m";
 function normalizeEvaluatorType(type) {
   return type.replace(/_/g, "-");
 }
+function isDeprecatedJudgeType(type) {
+  return type === "code-judge" || type === "llm-judge";
+}
 async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId) {
   const execution = rawEvalCase.execution;
   const executionObject = isJsonObject2(execution) ? execution : void 0;
@@ -15022,6 +15023,12 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
     const rawName = asString(rawEvaluator.name);
     const rawType = rawEvaluator.type;
     const typeValue = typeof rawType === "string" ? normalizeEvaluatorType(rawType) : rawType;
+    if (typeof typeValue === "string" && isDeprecatedJudgeType(typeValue)) {
+      logWarning2(
+        `Skipping evaluator '${rawName ?? "<unnamed>"}' in '${evalId}': '${rawType}' is deprecated. Use '${typeValue.replace("-judge", "-grader")}' instead`
+      );
+      continue;
+    }
     const isCustomType = typeof typeValue === "string" && !isEvaluatorKind(typeValue);
     if (typeof typeValue !== "string") {
       logWarning2(`Skipping evaluator with invalid type in '${evalId}'`);
@@ -15054,7 +15061,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "code-grader" || typeValue === "code-judge") {
+    if (typeValue === "code-grader") {
       let command;
       if (rawEvaluator.script !== void 0 && rawEvaluator.command === void 0) {
         console.warn(
@@ -15164,7 +15171,14 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
         continue;
       }
       const aggregatorType = asString(rawAggregator.type);
-      if (aggregatorType !== "weighted_average" && aggregatorType !== "code-grader" && aggregatorType !== "code-judge" && aggregatorType !== "llm-grader" && aggregatorType !== "llm-judge" && aggregatorType !== "threshold") {
+      const normalizedAggregatorType = typeof aggregatorType === "string" ? aggregatorType === "weighted_average" || aggregatorType === "threshold" ? aggregatorType : normalizeEvaluatorType(aggregatorType) : aggregatorType;
+      if (typeof normalizedAggregatorType === "string" && isDeprecatedJudgeType(normalizedAggregatorType)) {
+        logWarning2(
+          `Skipping composite evaluator '${name21}' in '${evalId}': aggregator type '${aggregatorType}' is deprecated. Use '${normalizedAggregatorType.replace("-judge", "-grader")}' instead`
+        );
+        continue;
+      }
+      if (normalizedAggregatorType !== "weighted_average" && normalizedAggregatorType !== "code-grader" && normalizedAggregatorType !== "llm-grader" && normalizedAggregatorType !== "threshold") {
         logWarning2(
           `Skipping composite evaluator '${name21}' in '${evalId}': invalid aggregator type '${aggregatorType}'`
         );
@@ -15199,7 +15213,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
         continue;
       }
       let aggregator;
-      if (aggregatorType === "weighted_average") {
+      if (normalizedAggregatorType === "weighted_average") {
         const weights = isJsonObject2(rawAggregator.weights) ? rawAggregator.weights : void 0;
         const parsedWeights = {};
         if (weights) {
@@ -15213,7 +15227,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
           type: "weighted_average",
           ...Object.keys(parsedWeights).length > 0 ? { weights: parsedWeights } : {}
         };
-      } else if (aggregatorType === "code-grader" || aggregatorType === "code-judge") {
+      } else if (normalizedAggregatorType === "code-grader") {
         const aggregatorPath = asString(rawAggregator.path);
         if (!aggregatorPath) {
           logWarning2(
@@ -15226,7 +15240,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
           path: aggregatorPath,
           cwd: searchRoots[0]
         };
-      } else if (aggregatorType === "threshold") {
+      } else if (normalizedAggregatorType === "threshold") {
         const thresholdValue = rawAggregator.threshold;
         if (typeof thresholdValue !== "number" || thresholdValue < 0 || thresholdValue > 1) {
           logWarning2(
@@ -15974,10 +15988,15 @@ function coerceEvaluator(candidate, contextId) {
     return void 0;
   }
   const normalized = normalizeEvaluatorType(candidate);
+  if (isDeprecatedJudgeType(normalized)) {
+    throw new Error(
+      `Unsupported grader '${candidate}' in ${contextId}. Use '${normalized.replace("-judge", "-grader")}' instead.`
+    );
+  }
   if (isEvaluatorKind(normalized)) {
     return normalized;
   }
-  logWarning2(`Unknown evaluator '${candidate}' in ${contextId}, falling back to default`);
+  logWarning2(`Unknown grader '${candidate}' in ${contextId}, falling back to default`);
   return void 0;
 }
 function asString(value) {
@@ -17380,9 +17399,7 @@ function assertionToNaturalLanguage(entry) {
     case "ends_with":
       return `Output ends with '${entry.value}'`;
     case "llm-grader":
-    case "llm_grader":
-    case "llm-judge":
-    case "llm_judge": {
+    case "llm_grader": {
       if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
         return null;
       }
@@ -17395,9 +17412,7 @@ function assertionToNaturalLanguage(entry) {
       return tools ? `Agent called tools in order: ${tools}` : "Agent followed expected tool trajectory";
     }
     case "code-grader":
-    case "code_grader":
-    case "code-judge":
-    case "code_judge": {
+    case "code_grader": {
       const graderName = entry.name ?? deriveGraderNameFromCommand(entry.command) ?? "code-grader";
       const desc = typeof entry.description === "string" ? entry.description : void 0;
       return codeGraderInstruction(graderName, desc);
@@ -17428,7 +17443,7 @@ function assertionToNaturalLanguage(entry) {
   }
 }
 function assertionToNaturalLanguageList(entry) {
-  if (entry.type === "llm-grader" || entry.type === "llm_grader" || entry.type === "llm-judge" || entry.type === "llm_judge") {
+  if (entry.type === "llm-grader" || entry.type === "llm_grader") {
     if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
       return entry.rubrics.map((r) => r.outcome ?? r.criteria ?? r.id).filter((s) => typeof s === "string");
     }
@@ -24084,7 +24099,7 @@ function toCamelCaseDeep(obj) {
 }
 var FILE_BACKED_OUTPUT_THRESHOLD = 5e4;
 var CodeEvaluator = class {
-  kind = "code-judge";
+  kind = "code-grader";
   command;
   cwd;
   agentTimeoutMs;
@@ -24103,7 +24118,7 @@ var CodeEvaluator = class {
     if (outputForPayload) {
       const serialized = JSON.stringify(outputForPayload);
       if (serialized.length > FILE_BACKED_OUTPUT_THRESHOLD) {
-        const tmpDir = await mkdtemp2(join(tmpdir2(), "agentv-judge-"));
+        const tmpDir = await mkdtemp2(join(tmpdir2(), "agentv-grader-"));
         outputPath = join(tmpDir, "output.json");
         await writeFile6(outputPath, serialized);
         outputForPayload = null;
@@ -24352,7 +24367,7 @@ var LlmGraderEvaluator = class {
       return this.evaluateWithDelegatedAgent(context2, graderProvider);
     }
     const config = context2.evaluator;
-    if ((config?.type === "llm-grader" || config?.type === "llm-judge") && config.rubrics && config.rubrics.length > 0) {
+    if (config?.type === "llm-grader" && config.rubrics && config.rubrics.length > 0) {
       return this.evaluateWithRubrics(context2, graderProvider, config.rubrics);
     }
     return this.evaluateFreeform(context2, graderProvider);
@@ -24537,7 +24552,7 @@ ${context2.fileChanges}`;
     const systemPrompt = this.buildAgentSystemPrompt(context2);
     const userPrompt = this.buildAgentUserPrompt(context2);
     const config = context2.evaluator;
-    const rubrics = config?.type === "llm-grader" || config?.type === "llm-judge" ? config.rubrics : void 0;
+    const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
     const fsTools = createFilesystemTools(workspacePath);
     const evaluatorRawRequest = {
       mode: "built-in",
@@ -24633,7 +24648,7 @@ ${context2.fileChanges}`;
         };
       }
       const config = context2.evaluator;
-      const rubrics = config?.type === "llm-grader" || config?.type === "llm-judge" ? config.rubrics : void 0;
+      const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
       const details = {
         mode: modeLabel,
         grader_target: provider.targetName
@@ -24673,7 +24688,7 @@ ${context2.fileChanges}`;
    */
   buildAgentSystemPrompt(context2) {
     const config = context2.evaluator;
-    const rubrics = config?.type === "llm-grader" || config?.type === "llm-judge" ? config.rubrics : void 0;
+    const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
     const parts = [
       "You are an expert evaluator with access to the workspace filesystem.",
       "Use the provided tools to investigate the workspace and verify the criteria are met.",
@@ -24704,7 +24719,7 @@ ${context2.fileChanges}`;
       return substituteVariables(this.evaluatorTemplate, variables);
     }
     const config = context2.evaluator;
-    const rubrics = config?.type === "llm-grader" || config?.type === "llm-judge" ? config.rubrics : void 0;
+    const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
     const parts = [
       "Evaluate the candidate answer by investigating the workspace.",
       "",
@@ -24747,7 +24762,7 @@ ${context2.fileChanges}`;
   buildDelegatedPrompt(context2) {
     const formattedQuestion = context2.promptInputs.question && context2.promptInputs.question.trim().length > 0 ? context2.promptInputs.question : context2.evalCase.question;
     const config = context2.evaluator;
-    const rubrics = config?.type === "llm-grader" || config?.type === "llm-judge" ? config.rubrics : void 0;
+    const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
     if (this.evaluatorTemplate) {
       const variables = {
         [TEMPLATE_VARIABLES.CRITERIA]: context2.evalCase.criteria.trim(),
@@ -25242,10 +25257,8 @@ var CompositeEvaluator = class {
     const aggregator = this.config.aggregator;
     switch (aggregator.type) {
       case "code-grader":
-      case "code-judge":
         return this.runCodeAggregator(results, aggregator.path, aggregator.cwd ?? this.cwd);
       case "llm-grader":
-      case "llm-judge":
         return this.runLlmAggregator(results, context2, aggregator);
       case "threshold":
         return this.runThreshold(results, aggregator.threshold);
@@ -27630,7 +27643,7 @@ var endsWithFactory = (config) => {
 };
 function createBuiltinRegistry() {
   const registry = new EvaluatorRegistry();
-  registry.register("llm-grader", llmGraderFactory).register("llm-judge", llmGraderFactory).register("code-grader", codeFactory).register("code-judge", codeFactory).register("composite", compositeFactory).register("tool-trajectory", toolTrajectoryFactory).register("field-accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token-usage", tokenUsageFactory).register("execution-metrics", executionMetricsFactory).register("skill-trigger", skillTriggerFactory).register("contains", containsFactory).register("contains-any", containsAnyFactory).register("contains-all", containsAllFactory).register("icontains", icontainsFactory).register("icontains-any", icontainsAnyFactory).register("icontains-all", icontainsAllFactory).register("starts-with", startsWithFactory).register("ends-with", endsWithFactory).register("regex", regexFactory).register("is-json", isJsonFactory).register("equals", equalsFactory).register("inline-assert", (config) => {
+  registry.register("llm-grader", llmGraderFactory).register("code-grader", codeFactory).register("composite", compositeFactory).register("tool-trajectory", toolTrajectoryFactory).register("field-accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token-usage", tokenUsageFactory).register("execution-metrics", executionMetricsFactory).register("skill-trigger", skillTriggerFactory).register("contains", containsFactory).register("contains-any", containsAnyFactory).register("contains-all", containsAllFactory).register("icontains", icontainsFactory).register("icontains-any", icontainsAnyFactory).register("icontains-all", icontainsAllFactory).register("starts-with", startsWithFactory).register("ends-with", endsWithFactory).register("regex", regexFactory).register("is-json", isJsonFactory).register("equals", equalsFactory).register("inline-assert", (config) => {
     const fn = config[INLINE_ASSERT_FN];
     if (!fn) {
       throw new Error(
@@ -30306,7 +30319,7 @@ function filterEvalCases(evalCases, filter2) {
   return evalCases.filter((evalCase) => micromatch3.isMatch(evalCase.id, filter2));
 }
 function buildEvaluatorRegistry(overrides, resolveGraderProvider) {
-  const llmGrader = overrides?.["llm-grader"] ?? overrides?.["llm-judge"] ?? new LlmGraderEvaluator({
+  const llmGrader = overrides?.["llm-grader"] ?? new LlmGraderEvaluator({
     resolveGraderProvider: async (context2) => {
       if (context2.graderProvider) {
         return context2.graderProvider;
@@ -31127,10 +31140,10 @@ var OtelTraceExporter = class {
         }
         if (result.scores) {
           for (const score of result.scores) {
-            rootSpan.addEvent(`agentv.evaluator.${score.name}`, {
-              "agentv.evaluator.score": score.score,
-              "agentv.evaluator.type": score.type,
-              ...score.verdict ? { "agentv.evaluator.verdict": score.verdict } : {}
+            rootSpan.addEvent(`agentv.grader.${score.name}`, {
+              "agentv.grader.score": score.score,
+              "agentv.grader.type": score.type,
+              ...score.verdict ? { "agentv.grader.verdict": score.verdict } : {}
             });
           }
         }
@@ -31590,4 +31603,4 @@ export {
   OtelStreamingObserver,
   createAgentKernel
 };
-//# sourceMappingURL=chunk-7OHZAFND.js.map
+//# sourceMappingURL=chunk-K747KGDP.js.map