@agentv/core 4.17.1 → 4.18.0-next.1

package/dist/index.js

@@ -13,7 +13,7 @@ import {
   isAgentProvider,
   isContent,
   isContentArray,
-  isEvaluatorKind,
+  isGraderKind,
   isJsonObject,
   isJsonValue,
   isTestMessage,
@@ -25,7 +25,7 @@ import {
   resolveDelegatedTargetDefinition,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-6VZY3B6M.js";
+} from "./chunk-PYDBJOAO.js";
 import {
   execFileWithStdin,
   execShellWithStdin
@@ -647,22 +647,25 @@ function extractCacheConfig(suite) {
   const resolvedCachePath = typeof cachePath === "string" && cachePath.trim().length > 0 ? cachePath.trim() : void 0;
   return { enabled: cache, cachePath: resolvedCachePath };
 }
-function extractTotalBudgetUsd(suite) {
+function extractBudgetUsd(suite) {
   const execution = suite.execution;
   if (!execution || typeof execution !== "object" || Array.isArray(execution)) {
     return void 0;
   }
   const executionObj = execution;
-  const rawBudget = executionObj.total_budget_usd ?? executionObj.totalBudgetUsd;
+  if ("total_budget_usd" in executionObj || "totalBudgetUsd" in executionObj) {
+    throw new Error(
+      "execution.total_budget_usd has been renamed to execution.budget_usd. Update your eval YAML."
+    );
+  }
+  const rawBudget = executionObj.budget_usd ?? executionObj.budgetUsd;
   if (rawBudget === void 0 || rawBudget === null) {
     return void 0;
   }
   if (typeof rawBudget === "number" && rawBudget > 0) {
     return rawBudget;
   }
-  logWarning(
-    `Invalid execution.total_budget_usd: ${rawBudget}. Must be a positive number. Ignoring.`
-  );
+  logWarning(`Invalid execution.budget_usd: ${rawBudget}. Must be a positive number. Ignoring.`);
   return void 0;
 }
 function extractFailOnError(suite) {
@@ -812,7 +815,7 @@ function logWarning(message) {
   console.warn(`${ANSI_YELLOW}Warning: ${message}${ANSI_RESET2}`);
 }
 
-// src/evaluation/loaders/evaluator-parser.ts
+// src/evaluation/loaders/grader-parser.ts
 import { readFile as readFile5 } from "node:fs/promises";
 import path5 from "node:path";
 import { parse as parse2 } from "yaml";
@@ -1051,38 +1054,38 @@ function validateTemplateVariables(content, source) {
     );
   }
   if (invalidVariables.length > 0) {
-    const warningMessage = `${ANSI_YELLOW2}Warning: Custom evaluator template at ${source}
+    const warningMessage = `${ANSI_YELLOW2}Warning: Custom grader template at ${source}
   Contains invalid variables: ${invalidVariables.map((v) => `{{ ${v} }}`).join(", ")}
   Valid variables: ${Array.from(VALID_TEMPLATE_VARIABLES).map((v) => `{{ ${v} }}`).join(", ")}${ANSI_RESET3}`;
     console.warn(warningMessage);
   }
 }
 
-// src/evaluation/loaders/evaluator-parser.ts
+// src/evaluation/loaders/grader-parser.ts
 var ANSI_YELLOW3 = "\x1B[33m";
 var ANSI_RESET4 = "\x1B[0m";
 var MAX_ASSERTION_INCLUDE_DEPTH = 3;
 var PROMPT_FILE_PREFIX = "file://";
-function normalizeEvaluatorType(type) {
+function normalizeGraderType(type) {
   return type.replace(/_/g, "-");
 }
 function isDeprecatedJudgeType(type) {
   return type === "code-judge" || type === "llm-judge";
 }
-async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId, defaultPreprocessors) {
+async function parseGraders(rawEvalCase, globalExecution, searchRoots, evalId, defaultPreprocessors) {
   const execution = rawEvalCase.execution;
   const executionObject = isJsonObject2(execution) ? execution : void 0;
   const caseEvaluators = rawEvalCase.assertions ?? rawEvalCase.assert ?? (executionObject ? executionObject.evaluators : void 0) ?? // deprecated: use assertions
   rawEvalCase.evaluators;
   const skipDefaults = executionObject?.skip_defaults === true;
   const rootEvaluators = skipDefaults ? void 0 : globalExecution?.assertions ?? globalExecution?.assert ?? globalExecution?.evaluators;
-  const parsedCase = await parseEvaluatorList(
+  const parsedCase = await parseGraderList(
     caseEvaluators,
     searchRoots,
     evalId,
     defaultPreprocessors
   );
-  const parsedRoot = await parseEvaluatorList(
+  const parsedRoot = await parseGraderList(
     rootEvaluators,
     searchRoots,
     evalId,
@@ -1161,12 +1164,12 @@ ${resolved.attempted.map((attempt) => `  Tried: ${attempt}`).join("\n")}` : "";
     templateDir,
     ...searchRoots.filter((root) => path5.resolve(root) !== templateDir)
   ];
-  return await expandEvaluatorEntries(assertions, nestedSearchRoots, evalId, {
+  return await expandGraderEntries(assertions, nestedSearchRoots, evalId, {
     depth: nextDepth,
     chain: [...includeContext.chain, resolved.resolvedPath]
   }) ?? [];
 }
-async function expandEvaluatorEntries(candidateEvaluators, searchRoots, evalId, includeContext = { depth: 0, chain: [] }) {
+async function expandGraderEntries(candidateEvaluators, searchRoots, evalId, includeContext = { depth: 0, chain: [] }) {
   if (candidateEvaluators === void 0) {
     return void 0;
   }
@@ -1190,8 +1193,8 @@ async function expandEvaluatorEntries(candidateEvaluators, searchRoots, evalId,
   }
   return expanded;
 }
-async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId, defaultPreprocessors) {
-  const expandedEvaluators = await expandEvaluatorEntries(candidateEvaluators, searchRoots, evalId);
+async function parseGraderList(candidateEvaluators, searchRoots, evalId, defaultPreprocessors) {
+  const expandedEvaluators = await expandGraderEntries(candidateEvaluators, searchRoots, evalId);
   if (!expandedEvaluators) {
     return void 0;
   }
@@ -1237,14 +1240,14 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId, defa
     }
     const rawName = asString(rawEvaluator.name);
     const rawType = rawEvaluator.type;
-    const typeValue = typeof rawType === "string" ? normalizeEvaluatorType(rawType) : rawType;
+    const typeValue = typeof rawType === "string" ? normalizeGraderType(rawType) : rawType;
     if (typeof typeValue === "string" && isDeprecatedJudgeType(typeValue)) {
       logWarning2(
         `Skipping evaluator '${rawName ?? "<unnamed>"}' in '${evalId}': '${rawType}' is deprecated. Use '${typeValue.replace("-judge", "-grader")}' instead`
       );
       continue;
     }
-    const isCustomType = typeof typeValue === "string" && !isEvaluatorKind(typeValue);
+    const isCustomType = typeof typeValue === "string" && !isGraderKind(typeValue);
     if (typeof typeValue !== "string") {
       logWarning2(`Skipping evaluator with invalid type in '${evalId}'`);
       continue;
@@ -1407,7 +1410,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId, defa
         continue;
       }
       const aggregatorType = asString(rawAggregator.type);
-      const normalizedAggregatorType = typeof aggregatorType === "string" ? aggregatorType === "weighted_average" || aggregatorType === "threshold" ? aggregatorType : normalizeEvaluatorType(aggregatorType) : aggregatorType;
+      const normalizedAggregatorType = typeof aggregatorType === "string" ? aggregatorType === "weighted_average" || aggregatorType === "threshold" ? aggregatorType : normalizeGraderType(aggregatorType) : aggregatorType;
       if (typeof normalizedAggregatorType === "string" && isDeprecatedJudgeType(normalizedAggregatorType)) {
         logWarning2(
           `Skipping composite evaluator '${name}' in '${evalId}': aggregator type '${aggregatorType}' is deprecated. Use '${normalizedAggregatorType.replace("-judge", "-grader")}' instead`
@@ -1420,7 +1423,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId, defa
         );
         continue;
       }
-      const expandedMembers = await expandEvaluatorEntries(
+      const expandedMembers = await expandGraderEntries(
         rawMembers,
         searchRoots,
         `${evalId}:${name}`
@@ -1436,11 +1439,11 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId, defa
         }
         const memberName = asString(rawMember.name);
         const memberType = rawMember.type;
-        if (!memberName || !isEvaluatorKind(memberType)) {
+        if (!memberName || !isGraderKind(memberType)) {
           logWarning2(`Skipping member evaluator with invalid name/type in composite '${name}'`);
           continue;
         }
-        const memberConfigs = await parseEvaluators(
+        const memberConfigs = await parseGraders(
           { evaluators: [rawMember] },
           void 0,
           searchRoots,
@@ -2181,7 +2184,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId, defa
         `prompt.command for evaluator '${name}' in '${evalId}'`
       );
       if (!commandArray) {
-        throw new Error(`Evaluator '${name}' in '${evalId}': prompt object requires command array`);
+        throw new Error(`Grader '${name}' in '${evalId}': prompt object requires command array`);
       }
       const commandPath = commandArray[commandArray.length - 1];
       const resolved = await resolveFileReference2(commandPath, searchRoots);
@@ -2189,7 +2192,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId, defa
         resolvedPromptScript = [...commandArray.slice(0, -1), path5.resolve(resolved.resolvedPath)];
       } else {
         throw new Error(
-          `Evaluator '${name}' in '${evalId}': prompt command file not found: ${resolved.displayPath}`
+          `Grader '${name}' in '${evalId}': prompt command file not found: ${resolved.displayPath}`
         );
       }
       if (isJsonObject2(rawPrompt.config)) {
@@ -2206,11 +2209,11 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId, defa
             await validateCustomPromptContent(promptPath);
           } catch (error) {
             const message = error instanceof Error ? error.message : String(error);
-            throw new Error(`Evaluator '${name}' template (${promptPath}): ${message}`);
+            throw new Error(`Grader '${name}' template (${promptPath}): ${message}`);
           }
         } else {
           throw new Error(
-            `Evaluator '${name}' in '${evalId}': prompt file not found: ${resolved.displayPath}`
+            `Grader '${name}' in '${evalId}': prompt file not found: ${resolved.displayPath}`
           );
         }
       } else {
@@ -2327,18 +2330,18 @@ async function parsePreprocessors(rawValue, searchRoots, evaluatorName, evalId)
     return void 0;
   }
   if (!Array.isArray(rawValue)) {
-    throw new Error(`Evaluator '${evaluatorName}' in '${evalId}': preprocessors must be an array`);
+    throw new Error(`Grader '${evaluatorName}' in '${evalId}': preprocessors must be an array`);
   }
   const preprocessors = [];
   for (const rawEntry of rawValue) {
     if (!isJsonObject2(rawEntry)) {
       throw new Error(
-        `Evaluator '${evaluatorName}' in '${evalId}': each preprocessor must be an object`
+        `Grader '${evaluatorName}' in '${evalId}': each preprocessor must be an object`
       );
     }
     const type = asString(rawEntry.type)?.trim();
     if (!type) {
-      throw new Error(`Evaluator '${evaluatorName}' in '${evalId}': preprocessor.type is required`);
+      throw new Error(`Grader '${evaluatorName}' in '${evalId}': preprocessor.type is required`);
     }
     const command = asStringArray(
       rawEntry.command,
@@ -2346,14 +2349,14 @@ async function parsePreprocessors(rawValue, searchRoots, evaluatorName, evalId)
     );
     if (!command || command.length === 0) {
       throw new Error(
-        `Evaluator '${evaluatorName}' in '${evalId}': preprocessor '${type}' requires command`
+        `Grader '${evaluatorName}' in '${evalId}': preprocessor '${type}' requires command`
       );
     }
     const commandPath = command[command.length - 1];
     const resolved = await resolveFileReference2(commandPath, searchRoots);
     if (!resolved.resolvedPath) {
       throw new Error(
-        `Evaluator '${evaluatorName}' in '${evalId}': preprocessor command file not found: ${resolved.displayPath}`
+        `Grader '${evaluatorName}' in '${evalId}': preprocessor command file not found: ${resolved.displayPath}`
       );
     }
     preprocessors.push({
@@ -2404,13 +2407,13 @@ function coerceEvaluator(candidate, contextId) {
   if (typeof candidate !== "string") {
     return void 0;
   }
-  const normalized = normalizeEvaluatorType(candidate);
+  const normalized = normalizeGraderType(candidate);
   if (isDeprecatedJudgeType(normalized)) {
     throw new Error(
       `Unsupported grader '${candidate}' in ${contextId}. Use '${normalized.replace("-judge", "-grader")}' instead.`
     );
   }
-  if (isEvaluatorKind(normalized)) {
+  if (isGraderKind(normalized)) {
     return normalized;
   }
   logWarning2(`Unknown grader '${candidate}' in ${contextId}, falling back to default`);
@@ -2482,7 +2485,7 @@ function parseRequiredAndMinScore(rawRequired, rawMinScore, evaluatorName, evalI
     }
     result.required = rawRequired;
     logWarning2(
-      `Evaluator '${evaluatorName}' in '${evalId}': 'required: ${rawRequired}' is deprecated. Use 'required: true' + 'min_score: ${rawRequired}' instead.`
+      `Grader '${evaluatorName}' in '${evalId}': 'required: ${rawRequired}' is deprecated. Use 'required: true' + 'min_score: ${rawRequired}' instead.`
     );
   }
   return result;
@@ -3302,7 +3305,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
     const testCaseEvaluatorKind = coerceEvaluator(testCaseConfig.evaluator, id) ?? globalEvaluator;
     let evaluators;
     try {
-      evaluators = await parseEvaluators(
+      evaluators = await parseGraders(
         testCaseConfig,
         mergedExecution,
         searchRoots,
@@ -3648,7 +3651,7 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
     targetRefs: extractTargetRefsFromSuite(parsed),
     workers: extractWorkersFromSuite(parsed),
     cacheConfig: extractCacheConfig(parsed),
-    totalBudgetUsd: extractTotalBudgetUsd(parsed),
+    budgetUsd: extractBudgetUsd(parsed),
     ...metadata !== void 0 && { metadata },
     ...failOnError !== void 0 && { failOnError },
     ...threshold !== void 0 && { threshold },
@@ -3789,7 +3792,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
     const testCaseEvaluatorKind = coerceEvaluator(testCaseConfig.evaluator, id) ?? globalEvaluator;
     let evaluators;
     try {
-      evaluators = await parseEvaluators(
+      evaluators = await parseGraders(
         testCaseConfig,
         globalExecution,
         searchRoots,
@@ -11600,7 +11603,7 @@ function resolveAndCreateProvider(definition, env = process.env) {
   return createProvider(resolved);
 }
 
-// src/evaluation/evaluators/scoring.ts
+// src/evaluation/graders/scoring.ts
 var DEFAULT_THRESHOLD = 0.8;
 var PASS_THRESHOLD = DEFAULT_THRESHOLD;
 function scoreToVerdict(score, threshold = DEFAULT_THRESHOLD) {
@@ -11688,7 +11691,7 @@ function negateScore(score) {
   };
 }
 
-// src/evaluation/evaluators/code-evaluator.ts
+// src/evaluation/graders/code-grader.ts
 import { mkdtemp as mkdtemp2, rm as rm3, writeFile as writeFile6 } from "node:fs/promises";
 import { tmpdir as tmpdir2 } from "node:os";
 import { dirname, join } from "node:path";
@@ -11981,7 +11984,7 @@ function getRepoCheckoutTargets(repos) {
   }));
 }
 
-// src/evaluation/evaluators/code-evaluator.ts
+// src/evaluation/graders/code-grader.ts
 var FILE_BACKED_OUTPUT_THRESHOLD = 5e4;
 var DATA_URI_RE = /^data:([^;]+);base64,(.+)$/s;
 async function materializeContentForGrader(messages, getWorkDir) {
@@ -12033,7 +12036,7 @@ async function materializeContentForGrader(messages, getWorkDir) {
   }
   return result;
 }
-var CodeEvaluator = class {
+var CodeGrader = class {
   kind = "code-grader";
   command;
   cwd;
@@ -12151,7 +12154,7 @@ var CodeEvaluator = class {
       })) : [];
       const details = parsed?.details && typeof parsed.details === "object" && !Array.isArray(parsed.details) ? parsed.details : void 0;
       const proxyUsage = getProxyUsage?.();
-      const evaluatorRawRequest = {
+      const graderRawRequest = {
         command: this.command,
         ...this.cwd ? { cwd: this.cwd } : {},
         ...proxyUsage ? {
@@ -12166,7 +12169,7 @@ var CodeEvaluator = class {
         verdict: scoreToVerdict(score),
         assertions,
         expectedAspectCount: assertions.length || 1,
-        evaluatorRawRequest,
+        graderRawRequest,
         ...details ? { details } : {},
         tokenUsage: proxyUsage?.tokenUsage
       };
@@ -12178,7 +12181,7 @@ var CodeEvaluator = class {
         verdict: "fail",
         assertions: [{ text: `Code evaluator failed: ${message}`, passed: false }],
         expectedAspectCount: 1,
-        evaluatorRawRequest: {
+        graderRawRequest: {
           command: this.command,
           ...this.cwd ? { cwd: this.cwd } : {},
           ...proxyUsage ? {
@@ -12227,10 +12230,10 @@ function formatStderr(stderr) {
 ${tail}`;
 }
 
-// src/evaluation/evaluators/composite.ts
+// src/evaluation/graders/composite.ts
 import { generateText as generateText3 } from "ai";
 
-// src/evaluation/evaluators/llm-grader.ts
+// src/evaluation/graders/llm-grader.ts
 import fs2 from "node:fs/promises";
 import path37 from "node:path";
 import { generateText as generateText2, stepCountIs, tool } from "ai";
@@ -12270,7 +12273,7 @@ var BINARY_EXTENSIONS = /* @__PURE__ */ new Set([
   ".so",
   ".dylib"
 ]);
-var DEFAULT_EVALUATOR_TEMPLATE = `You are an expert evaluator. Your goal is to grade the answer based on how well it achieves the criteria for the original task.
+var DEFAULT_GRADER_TEMPLATE = `You are an expert grader. Your goal is to grade the answer based on how well it achieves the criteria for the original task.
 
 Use the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The answer does not need to match it verbatim, but should capture the key points and follow the same spirit.
 
@@ -12325,19 +12328,19 @@ function resolveContentBasePath(context) {
   }
   return void 0;
 }
-var LlmGraderEvaluator = class {
+var LlmGrader = class {
   kind = "llm-grader";
   resolveGraderProvider;
   maxOutputTokens;
   temperature;
-  evaluatorTemplate;
+  graderTemplate;
   maxSteps;
   graderTargetProvider;
   constructor(options) {
     this.resolveGraderProvider = options.resolveGraderProvider ?? options.resolveJudgeProvider;
     this.maxOutputTokens = options.maxOutputTokens;
     this.temperature = options.temperature;
-    this.evaluatorTemplate = options.evaluatorTemplate;
+    this.graderTemplate = options.graderTemplate;
     this.maxSteps = Math.min(options.maxSteps ?? DEFAULT_MAX_STEPS, MAX_STEPS_LIMIT);
     this.graderTargetProvider = options.graderTargetProvider ?? options.judgeTargetProvider;
   }
@@ -12400,16 +12403,16 @@ var LlmGraderEvaluator = class {
       [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context.evalCase.reference_answer ?? "").trim()
     };
     const systemPrompt = buildOutputSchema();
-    const evaluatorTemplate = context.evaluatorTemplateOverride ?? this.evaluatorTemplate ?? DEFAULT_EVALUATOR_TEMPLATE;
-    warnDeprecatedTemplateVars(evaluatorTemplate);
-    let userPrompt = substituteVariables(evaluatorTemplate, variables);
-    if (context.fileChanges && !context.evaluatorTemplateOverride && !this.evaluatorTemplate) {
+    const graderTemplate = context.graderTemplateOverride ?? this.graderTemplate ?? DEFAULT_GRADER_TEMPLATE;
+    warnDeprecatedTemplateVars(graderTemplate);
+    let userPrompt = substituteVariables(graderTemplate, variables);
+    if (context.fileChanges && !context.graderTemplateOverride && !this.graderTemplate) {
       userPrompt += `
 
 [[ ## file_changes ## ]]
 ${context.fileChanges}`;
     }
-    const evaluatorRawRequest = {
+    const graderRawRequest = {
       userPrompt,
       systemPrompt
     };
@@ -12430,7 +12433,7 @@ ${context.fileChanges}`;
         verdict: scoreToVerdict(score),
         assertions,
         expectedAspectCount: Math.max(assertions.length, 1),
-        evaluatorRawRequest,
+        graderRawRequest,
         graderTarget: graderProvider.targetName,
         details: data.details,
         tokenUsage
@@ -12444,7 +12447,7 @@ ${context.fileChanges}`;
         verdict: "skip",
         assertions: [{ text: `Grader parse failure after 3 attempts: ${message}`, passed: false }],
         expectedAspectCount: 1,
-        evaluatorRawRequest,
+        graderRawRequest,
         graderTarget: graderProvider.targetName
       };
     }
@@ -12461,7 +12464,7 @@ ${context.fileChanges}`;
     }
     const prompt = this.buildRubricPrompt(context, rubrics);
     const systemPrompt = buildRubricOutputSchema();
-    const evaluatorRawRequest = {
+    const graderRawRequest = {
       userPrompt: prompt,
       systemPrompt
     };
@@ -12481,7 +12484,7 @@ ${context.fileChanges}`;
         verdict,
         assertions,
         expectedAspectCount: rubrics.length,
-        evaluatorRawRequest,
+        graderRawRequest,
         graderTarget: graderProvider.targetName,
         tokenUsage
       };
@@ -12494,7 +12497,7 @@ ${context.fileChanges}`;
         verdict: "skip",
         assertions: [{ text: `Grader parse failure after 3 attempts: ${message}`, passed: false }],
         expectedAspectCount: rubrics.length,
-        evaluatorRawRequest,
+        graderRawRequest,
         graderTarget: graderProvider.targetName
       };
     }
@@ -12506,7 +12509,7 @@ ${context.fileChanges}`;
   async evaluateWithScoreRanges(context, graderProvider, rubrics) {
     const prompt = this.buildScoreRangePrompt(context, rubrics);
     const systemPrompt = buildScoreRangeOutputSchema();
-    const evaluatorRawRequest = {
+    const graderRawRequest = {
       userPrompt: prompt,
       systemPrompt
     };
@@ -12526,7 +12529,7 @@ ${context.fileChanges}`;
         verdict,
         assertions,
         expectedAspectCount: rubrics.length,
-        evaluatorRawRequest,
+        graderRawRequest,
         graderTarget: graderProvider.targetName,
         details,
         tokenUsage
@@ -12540,7 +12543,7 @@ ${context.fileChanges}`;
         verdict: "skip",
         assertions: [{ text: `Grader parse failure after 3 attempts: ${message}`, passed: false }],
         expectedAspectCount: rubrics.length,
-        evaluatorRawRequest,
+        graderRawRequest,
         graderTarget: graderProvider.targetName
       };
     }
@@ -12569,7 +12572,7 @@ ${context.fileChanges}`;
     const config = context.evaluator;
     const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
     const fsTools = createFilesystemTools(workspacePath);
-    const evaluatorRawRequest = {
+    const graderRawRequest = {
       mode: "built-in",
       systemPrompt,
       userPrompt,
@@ -12593,7 +12596,7 @@ ${context.fileChanges}`;
       return this.parseAgentResult(
         text,
         rubrics,
-        evaluatorRawRequest,
+        graderRawRequest,
         details,
         graderProvider.targetName
       );
@@ -12604,7 +12607,7 @@ ${context.fileChanges}`;
         verdict: "fail",
         assertions: [{ text: `llm-grader built-in evaluation failed: ${message}`, passed: false }],
         expectedAspectCount: 1,
-        evaluatorRawRequest,
+        graderRawRequest,
         graderTarget: graderProvider.targetName,
         details: { mode: "built-in", error: message }
       };
@@ -12636,7 +12639,7 @@ ${context.fileChanges}`;
   async evaluateWithDelegate(context, provider, modeLabel) {
     const workspacePath = context.workspacePath;
     const prompt = this.buildDelegatedPrompt(context);
-    const evaluatorRawRequest = {
+    const graderRawRequest = {
       mode: modeLabel,
       grader_target: provider.targetName,
       prompt
@@ -12657,7 +12660,7 @@ ${context.fileChanges}`;
             { text: `llm-grader ${modeLabel} returned no assistant response`, passed: false }
           ],
           expectedAspectCount: 1,
-          evaluatorRawRequest,
+          graderRawRequest,
           graderTarget: provider.targetName,
           details: { mode: modeLabel, grader_target: provider.targetName }
         };
@@ -12671,7 +12674,7 @@ ${context.fileChanges}`;
       return this.parseAgentResult(
         assistantContent,
         rubrics,
-        evaluatorRawRequest,
+        graderRawRequest,
         details,
         provider.targetName
       );
@@ -12684,7 +12687,7 @@ ${context.fileChanges}`;
           { text: `llm-grader ${modeLabel} evaluation failed: ${message}`, passed: false }
         ],
         expectedAspectCount: 1,
-        evaluatorRawRequest,
+        graderRawRequest,
         graderTarget: provider.targetName,
         details: {
           mode: modeLabel,
@@ -12705,7 +12708,7 @@ ${context.fileChanges}`;
     const config = context.evaluator;
     const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
     const parts = [
-      "You are an expert evaluator with access to the workspace filesystem.",
+      "You are an expert grader with access to the workspace filesystem.",
       "Use the provided tools to investigate the workspace and verify the criteria are met.",
       "Thoroughly examine relevant files before making your assessment.",
       ""
@@ -12734,9 +12737,9 @@ ${context.fileChanges}`;
       [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context.candidate.trim(),
       [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context.evalCase.reference_answer ?? "").trim()
     };
-    if (this.evaluatorTemplate) {
-      warnDeprecatedTemplateVars(this.evaluatorTemplate);
-      return substituteVariables(this.evaluatorTemplate, variables);
+    if (this.graderTemplate) {
+      warnDeprecatedTemplateVars(this.graderTemplate);
+      return substituteVariables(this.graderTemplate, variables);
     }
     const config = context.evaluator;
     const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
@@ -12783,7 +12786,7 @@ ${context.fileChanges}`;
     const formattedQuestion = context.promptInputs.question && context.promptInputs.question.trim().length > 0 ? context.promptInputs.question : context.evalCase.question;
     const config = context.evaluator;
     const rubrics = config?.type === "llm-grader" ? config.rubrics : void 0;
-    if (this.evaluatorTemplate) {
+    if (this.graderTemplate) {
       const variables = {
         [TEMPLATE_VARIABLES.CRITERIA]: context.evalCase.criteria.trim(),
         [TEMPLATE_VARIABLES.INPUT]: formattedQuestion.trim(),
@@ -12795,15 +12798,15 @@ ${context.fileChanges}`;
         [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context.candidate.trim(),
         [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context.evalCase.reference_answer ?? "").trim()
       };
-      warnDeprecatedTemplateVars(this.evaluatorTemplate);
-      const customPrompt = substituteVariables(this.evaluatorTemplate, variables);
+      warnDeprecatedTemplateVars(this.graderTemplate);
+      const customPrompt = substituteVariables(this.graderTemplate, variables);
       const outputSchema = rubrics && rubrics.length > 0 ? buildRubricOutputSchema() : buildOutputSchema();
       return `${customPrompt}
 
 ${outputSchema}`;
     }
     const parts = [
-      "You are an expert evaluator. Investigate the workspace to verify the criteria are met.",
+      "You are an expert grader. Investigate the workspace to verify the criteria are met.",
       "",
       "[[ ## question ## ]]",
       formattedQuestion,
@@ -12840,7 +12843,7 @@ ${outputSchema}`;
    * Parse the agent's response text into an EvaluationScore.
    * Supports both freeform and rubric modes.
    */
-  parseAgentResult(text, rubrics, evaluatorRawRequest, details, graderTarget) {
+  parseAgentResult(text, rubrics, graderRawRequest, details, graderTarget) {
     try {
       const parsed = parseJsonFromText(text);
       if (rubrics && rubrics.length > 0) {
@@ -12851,7 +12854,7 @@ ${outputSchema}`;
           verdict,
           assertions: assertions2,
           expectedAspectCount: rubrics.length,
-          evaluatorRawRequest,
+          graderRawRequest,
           graderTarget,
           details
         };
@@ -12864,7 +12867,7 @@ ${outputSchema}`;
         verdict: scoreToVerdict(score),
         assertions,
         expectedAspectCount: Math.max(assertions.length, 1),
-        evaluatorRawRequest,
+        graderRawRequest,
         graderTarget,
         details: data.details && Object.keys(data.details).length > 0 ? { ...details, ...data.details } : details
       };
@@ -12879,7 +12882,7 @@ ${outputSchema}`;
           }
         ],
         expectedAspectCount: 1,
-        evaluatorRawRequest,
+        graderRawRequest,
         graderTarget,
         details
       };
@@ -12894,7 +12897,7 @@ ${outputSchema}`;
   buildScoreRangePrompt(context, rubrics) {
     const formattedQuestion = context.promptInputs.question && context.promptInputs.question.trim().length > 0 ? context.promptInputs.question : context.evalCase.question;
     const parts = [
-      "You are an expert evaluator. Score the candidate answer on each criterion below using the provided score ranges.",
+      "You are an expert grader. Score the candidate answer on each criterion below using the provided score ranges.",
       "For each criterion, output an integer score from 0 to 10 based on which score range best matches the answer.",
       "",
       "[[ ## question ## ]]",
@@ -12937,7 +12940,7 @@ ${outputSchema}`;
   buildRubricPrompt(context, rubrics) {
     const formattedQuestion = context.promptInputs.question && context.promptInputs.question.trim().length > 0 ? context.promptInputs.question : context.evalCase.question;
     const parts = [
-      "You are an expert evaluator. Evaluate the candidate answer against each rubric item below.",
+      "You are an expert grader. Evaluate the candidate answer against each rubric item below.",
       "",
       "[[ ## question ## ]]",
       formattedQuestion,
@@ -13111,7 +13114,7 @@ function sumTokenUsage(first, second) {
   };
 }
 function buildRubricOutputSchema() {
-  return `You are an expert evaluator. Evaluate the candidate answer against each rubric item.
+  return `You are an expert grader. Evaluate the candidate answer against each rubric item.
 You must return a valid JSON object matching this schema:
 {
   "checks": [
@@ -13145,7 +13148,7 @@ function warnDeprecatedTemplateVars(template) {
     console.warn(
       `${ANSI_YELLOW7}\u26A0 Deprecated template variables detected (they still work but will be removed in a future version):
   ${used.join("\n  ")}
-  Update your custom evaluator template to use the new names.${ANSI_RESET8}`
+  Update your custom grader template to use the new names.${ANSI_RESET8}`
     );
   }
 }
@@ -13177,7 +13180,7 @@ function calculateRubricScore(result, rubrics) {
   return { score, verdict, assertions };
 }
 function buildScoreRangeOutputSchema() {
-  return `You are an expert evaluator. Score the candidate answer on each criterion.
+  return `You are an expert grader. Score the candidate answer on each criterion.
 You must return a valid JSON object matching this schema:
 {
   "checks": [
@@ -13385,13 +13388,13 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
   }
 }
 
-// src/evaluation/evaluators/composite.ts
+// src/evaluation/graders/composite.ts
 var DEFAULT_COMPOSITE_AGGREGATOR_PROMPT = `Review the following evaluation results:
 {{EVALUATOR_RESULTS_JSON}}
 
-Decide the final score and verdict based on all evaluator results.
+Decide the final score and verdict based on all grader results.
 Return a JSON object with: score (0.0-1.0), verdict (pass/fail), and reasoning.`;
-var CompositeEvaluator = class {
+var CompositeGrader = class {
   kind = "composite";
   config;
   evaluatorFactory;
@@ -13442,7 +13445,7 @@ var CompositeEvaluator = class {
         weight,
         verdict: member.result.verdict,
         assertions: [...member.result.assertions],
-        evaluatorRawRequest: member.result.evaluatorRawRequest,
+        graderRawRequest: member.result.graderRawRequest,
         scores: member.result.scores,
         details: member.result.details,
         tokenUsage: member.result.tokenUsage
@@ -13463,7 +13466,7 @@ var CompositeEvaluator = class {
         verdict: "skip",
         assertions: [{ text: "All evaluators skipped (infrastructure failure)", passed: false }],
         expectedAspectCount: 1,
-        evaluatorRawRequest: {
+        graderRawRequest: {
           aggregator: "weighted_average",
           ...weights ? { weights } : {}
         },
@@ -13476,7 +13479,7 @@ var CompositeEvaluator = class {
       verdict: scoreToVerdict(finalScore),
       assertions: allAssertions,
       expectedAspectCount: allAssertions.length || 1,
-      evaluatorRawRequest: {
+      graderRawRequest: {
         aggregator: "weighted_average",
         ...weights ? { weights } : {}
       },
@@ -13495,7 +13498,7 @@ var CompositeEvaluator = class {
         score: member.result.score,
         verdict: member.result.verdict,
         assertions: [...member.result.assertions],
-        evaluatorRawRequest: member.result.evaluatorRawRequest,
+        graderRawRequest: member.result.graderRawRequest,
         scores: member.result.scores,
         details: member.result.details,
         tokenUsage: member.result.tokenUsage
@@ -13518,7 +13521,7 @@ var CompositeEvaluator = class {
         verdict: "skip",
         assertions: [{ text: "All evaluators skipped (infrastructure failure)", passed: false }],
         expectedAspectCount: 1,
-        evaluatorRawRequest: {
+        graderRawRequest: {
           aggregator: "threshold",
           threshold
         },
@@ -13537,7 +13540,7 @@ var CompositeEvaluator = class {
       verdict: pass ? "pass" : "fail",
       assertions: allAssertions,
       expectedAspectCount: allAssertions.length || 1,
-      evaluatorRawRequest: {
+      graderRawRequest: {
         aggregator: "threshold",
         threshold
       },
@@ -13554,7 +13557,7 @@ var CompositeEvaluator = class {
       weight: weights?.[member.id] ?? 1,
       verdict: member.result.verdict,
       assertions: [...member.result.assertions],
-      evaluatorRawRequest: member.result.evaluatorRawRequest,
+      graderRawRequest: member.result.graderRawRequest,
       scores: member.result.scores,
       details: member.result.details
     }));
@@ -13575,7 +13578,7 @@ var CompositeEvaluator = class {
         verdict,
         assertions,
         expectedAspectCount: assertions.length || 1,
-        evaluatorRawRequest: {
+        graderRawRequest: {
           aggregator: "code-grader",
           script: scriptPath
         },
@@ -13588,7 +13591,7 @@ var CompositeEvaluator = class {
         verdict: "fail",
         assertions: [{ text: `Code aggregator failed: ${message}`, passed: false }],
         expectedAspectCount: 1,
-        evaluatorRawRequest: {
+        graderRawRequest: {
           aggregator: "code-grader",
           script: scriptPath,
           error: message
@@ -13610,14 +13613,14 @@ var CompositeEvaluator = class {
       score: member.result.score,
       verdict: member.result.verdict,
       assertions: [...member.result.assertions],
-      evaluatorRawRequest: member.result.evaluatorRawRequest,
+      graderRawRequest: member.result.graderRawRequest,
       scores: member.result.scores,
       details: member.result.details
     }));
     const promptTemplate = config.prompt ?? DEFAULT_COMPOSITE_AGGREGATOR_PROMPT;
     const userPrompt = promptTemplate.replace(/\{\{EVALUATOR_RESULTS_JSON\}\}/g, resultsJson);
     const systemPrompt = buildOutputSchema();
-    const evaluatorRawRequest = {
+    const graderRawRequest = {
       aggregator: "llm-grader",
       userPrompt,
       systemPrompt,
@@ -13639,7 +13642,7 @@ var CompositeEvaluator = class {
           verdict: scoreToVerdict(score2),
           assertions: assertions2,
           expectedAspectCount: Math.max(assertions2.length, 1),
-          evaluatorRawRequest,
+          graderRawRequest,
           scores
         };
       }
@@ -13659,7 +13662,7 @@ var CompositeEvaluator = class {
         verdict: scoreToVerdict(score),
         assertions,
         expectedAspectCount: Math.max(assertions.length, 1),
-        evaluatorRawRequest,
+        graderRawRequest,
         scores
       };
     } catch {
@@ -13668,15 +13671,15 @@ var CompositeEvaluator = class {
         verdict: "fail",
         assertions: [{ text: "LLM aggregator failed", passed: false }],
         expectedAspectCount: 1,
-        evaluatorRawRequest,
+        graderRawRequest,
         scores
       };
     }
   }
 };
 
-// src/evaluation/evaluators/cost.ts
-var CostEvaluator = class {
+// src/evaluation/graders/cost.ts
+var CostGrader = class {
   kind = "cost";
   config;
   constructor(options) {
@@ -13691,7 +13694,7 @@ var CostEvaluator = class {
         verdict: "fail",
         assertions: [{ text: "No cost data available in trace", passed: false }],
         expectedAspectCount: 1,
-        evaluatorRawRequest: {
+        graderRawRequest: {
           type: "cost",
           budget,
           costUsd: null
@@ -13708,7 +13711,7 @@ var CostEvaluator = class {
         passed ? { text: `Cost ${formatCost(costUsd)} <= ${formatCost(budget)} budget`, passed: true } : { text: `Cost ${formatCost(costUsd)} > ${formatCost(budget)} budget`, passed: false }
       ],
       expectedAspectCount: 1,
-      evaluatorRawRequest: {
+      graderRawRequest: {
         type: "cost",
         budget,
         costUsd
@@ -13717,8 +13720,8 @@ var CostEvaluator = class {
   }
 };
 
-// src/evaluation/evaluators/execution-metrics.ts
-var ExecutionMetricsEvaluator = class {
+// src/evaluation/graders/execution-metrics.ts
+var ExecutionMetricsGrader = class {
   kind = "execution-metrics";
   config;
   constructor(options) {
@@ -13742,7 +13745,7 @@ var ExecutionMetricsEvaluator = class {
         verdict: "fail",
         assertions: [{ text: "No trace summary available", passed: false }],
         expectedAspectCount: 1,
-        evaluatorRawRequest: {
+        graderRawRequest: {
           type: "execution-metrics",
           config: this.extractConfiguredThresholds(),
           actual: null
@@ -13858,7 +13861,7 @@ var ExecutionMetricsEvaluator = class {
       verdict: scoreToVerdict(score),
       assertions,
       expectedAspectCount: totalChecks || 1,
-      evaluatorRawRequest: {
+      graderRawRequest: {
         type: "execution-metrics",
         config: this.extractConfiguredThresholds(),
         actual: this.filterDefinedMetrics(actualMetrics)
@@ -13901,7 +13904,7 @@ var ExecutionMetricsEvaluator = class {
   }
 };
 
-// src/evaluation/evaluators/field-accuracy.ts
+// src/evaluation/graders/field-accuracy.ts
 var DEFAULT_DATE_FORMATS = [
   "YYYY-MM-DDTHH:mm:ssZ",
   // ISO with timezone
@@ -13946,7 +13949,7 @@ var MONTH_NAMES = {
   dec: 11,
   december: 11
 };
-var FieldAccuracyEvaluator = class {
+var FieldAccuracyGrader = class {
   kind = "field-accuracy";
   config;
   constructor(options) {
@@ -14305,8 +14308,8 @@ function parseJsonFromTextSafe(text) {
   return parseJsonFromText(text);
 }
 
-// src/evaluation/evaluators/latency.ts
-var LatencyEvaluator = class {
+// src/evaluation/graders/latency.ts
+var LatencyGrader = class {
   kind = "latency";
   config;
   constructor(options) {
@@ -14321,7 +14324,7 @@ var LatencyEvaluator = class {
         verdict: "fail",
         assertions: [{ text: "No duration data available in trace", passed: false }],
         expectedAspectCount: 1,
-        evaluatorRawRequest: {
+        graderRawRequest: {
           type: "latency",
           threshold,
           durationMs: null
@@ -14337,7 +14340,7 @@ var LatencyEvaluator = class {
         passed ? { text: `Duration ${durationMs}ms <= ${threshold}ms threshold`, passed: true } : { text: `Duration ${durationMs}ms > ${threshold}ms threshold`, passed: false }
       ],
       expectedAspectCount: 1,
-      evaluatorRawRequest: {
+      graderRawRequest: {
         type: "latency",
         threshold,
         durationMs
@@ -14346,8 +14349,8 @@ var LatencyEvaluator = class {
   }
 };
 
-// src/evaluation/evaluators/skill-trigger.ts
-var SkillTriggerEvaluator = class {
+// src/evaluation/graders/skill-trigger.ts
+var SkillTriggerGrader = class {
   kind = "skill-trigger";
   config;
   constructor(config) {
@@ -14414,7 +14417,7 @@ var SkillTriggerEvaluator = class {
   }
 };
 
-// src/evaluation/evaluators/llm-grader-prompt.ts
+// src/evaluation/graders/llm-grader-prompt.ts
 function assembleLlmGraderPrompt(input) {
   const {
     evalCase,
@@ -14422,7 +14425,7 @@ function assembleLlmGraderPrompt(input) {
     promptInputs,
     evaluatorConfig,
     fileChanges,
-    evaluatorTemplateOverride
+    graderTemplateOverride
   } = input;
   const rubrics = evaluatorConfig?.rubrics;
   if (rubrics && rubrics.length > 0) {
@@ -14432,15 +14435,9 @@ function assembleLlmGraderPrompt(input) {
     }
     return assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges);
   }
-  return assembleFreeform(
-    evalCase,
-    candidate,
-    promptInputs,
-    fileChanges,
-    evaluatorTemplateOverride
-  );
+  return assembleFreeform(evalCase, candidate, promptInputs, fileChanges, graderTemplateOverride);
 }
-function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, evaluatorTemplateOverride) {
+function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, graderTemplateOverride) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const variables = {
     [TEMPLATE_VARIABLES.INPUT]: formattedQuestion.trim(),
@@ -14454,9 +14451,9 @@ function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, evalua
     [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (evalCase.reference_answer ?? "").trim()
   };
   const systemPrompt = buildOutputSchema();
-  const template = evaluatorTemplateOverride ?? DEFAULT_EVALUATOR_TEMPLATE;
+  const template = graderTemplateOverride ?? DEFAULT_GRADER_TEMPLATE;
   let userPrompt = substituteVariables(template, variables);
-  if (fileChanges && !evaluatorTemplateOverride) {
+  if (fileChanges && !graderTemplateOverride) {
     userPrompt += `
 
 [[ ## file_changes ## ]]
@@ -14472,7 +14469,7 @@ ${fileChanges}`;
 function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const parts = [
-    "You are an expert evaluator. Evaluate the candidate answer against each rubric item below.",
+    "You are an expert grader. Evaluate the candidate answer against each rubric item below.",
     "",
     "[[ ## question ## ]]",
     formattedQuestion,
@@ -14507,7 +14504,7 @@ function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChang
 function assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChanges) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const parts = [
-    "You are an expert evaluator. Score the candidate answer on each criterion below using the provided score ranges.",
+    "You are an expert grader. Score the candidate answer on each criterion below using the provided score ranges.",
     "For each criterion, output an integer score from 0 to 10 based on which score range best matches the answer.",
     "",
     "[[ ## question ## ]]",
@@ -14555,8 +14552,8 @@ function assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChan
   };
 }
 
-// src/evaluation/evaluators/token-usage.ts
-var TokenUsageEvaluator = class {
+// src/evaluation/graders/token-usage.ts
+var TokenUsageGrader = class {
   kind = "token-usage";
   config;
   constructor(options) {
@@ -14577,7 +14574,7 @@ var TokenUsageEvaluator = class {
         verdict: "fail",
         assertions: [{ text: "No token usage data available in trace", passed: false }],
         expectedAspectCount,
-        evaluatorRawRequest: {
+        graderRawRequest: {
           type: "token-usage",
           max_total: maxTotal ?? null,
           max_input: maxInput ?? null,
@@ -14618,7 +14615,7 @@ var TokenUsageEvaluator = class {
       verdict: passed ? "pass" : "fail",
       assertions,
       expectedAspectCount,
-      evaluatorRawRequest: {
+      graderRawRequest: {
         type: "token-usage",
         max_total: maxTotal ?? null,
         max_input: maxInput ?? null,
@@ -14634,7 +14631,7 @@ var TokenUsageEvaluator = class {
   }
 };
 
-// src/evaluation/evaluators/tool-trajectory.ts
+// src/evaluation/graders/tool-trajectory.ts
 function getNestedValue(obj, path53) {
   const parts = path53.split(".");
   let current = obj;
@@ -14703,7 +14700,7 @@ function checkLatency(toolName, maxDurationMs, actualDurationMs) {
     message: `${toolName} took ${actualDurationMs}ms (max: ${maxDurationMs}ms)`
   };
 }
-var ToolTrajectoryEvaluator = class {
+var ToolTrajectoryGrader = class {
   kind = "tool-trajectory";
   config;
   constructor(options) {
@@ -15108,7 +15105,7 @@ var ToolTrajectoryEvaluator = class {
   }
 };
 
-// src/evaluation/evaluators/assertions.ts
+// src/evaluation/graders/assertions.ts
 function runContainsAssertion(output, value) {
   const passed = output.includes(value);
   return {
@@ -15403,15 +15400,15 @@ function validateConcurrency(concurrency) {
   }
 }
 
-// src/evaluation/registry/evaluator-registry.ts
-var EvaluatorRegistry = class {
+// src/evaluation/registry/grader-registry.ts
+var GraderRegistry = class {
   factories = /* @__PURE__ */ new Map();
-  /** Register a factory function for an evaluator type. */
+  /** Register a factory function for an grader type. */
   register(type, factory) {
     this.factories.set(type, factory);
     return this;
   }
-  /** Get the factory function for an evaluator type. */
+  /** Get the factory function for an grader type. */
   get(type) {
     return this.factories.get(type);
   }
@@ -15419,25 +15416,25 @@ var EvaluatorRegistry = class {
   has(type) {
     return this.factories.has(type);
   }
-  /** List all registered evaluator type names. */
+  /** List all registered grader type names. */
   list() {
     return [...this.factories.keys()];
   }
   /**
    * Create an evaluator instance from a config, using the registered factory.
-   * Throws if no factory is registered for the evaluator type.
+   * Throws if no factory is registered for the grader type.
    */
   async create(config, context) {
     const factory = this.factories.get(config.type);
     if (!factory) {
       throw new Error(
-        `Unknown evaluator type: "${config.type}". Registered types: ${this.list().join(", ")}`
+        `Unknown grader type: "${config.type}". Registered types: ${this.list().join(", ")}`
       );
     }
     return factory(config, context);
   }
 };
-var DeterministicAssertionEvaluator = class {
+var DeterministicAssertionGrader = class {
   constructor(kind, assertFn) {
     this.assertFn = assertFn;
     this.kind = kind;
@@ -15448,8 +15445,8 @@ var DeterministicAssertionEvaluator = class {
   }
 };
 
-// src/evaluation/evaluators/inline-assert.ts
-var InlineAssertEvaluator = class {
+// src/evaluation/graders/inline-assert.ts
+var InlineAssertGrader = class {
   constructor(fn, name) {
     this.fn = fn;
     this.name = name;
@@ -15474,7 +15471,7 @@ var InlineAssertEvaluator = class {
   }
 };
 
-// src/evaluation/evaluators/prompt-resolution.ts
+// src/evaluation/graders/prompt-resolution.ts
 import path38 from "node:path";
 async function resolveCustomPrompt(promptConfig, context, timeoutMs) {
   if (promptConfig.resolvedPromptScript && promptConfig.resolvedPromptScript.length > 0) {
@@ -15541,7 +15538,7 @@ async function executePromptTemplate(script, context, config, timeoutMs) {
   }
 }
 
-// src/evaluation/registry/builtin-evaluators.ts
+// src/evaluation/registry/builtin-graders.ts
 var INLINE_ASSERT_FN = Symbol.for("agentv.inline-assert-fn");
 var llmGraderFactory = (config, context) => {
   const c = config;
@@ -15558,7 +15555,7 @@ var llmGraderFactory = (config, context) => {
       );
     }
     const isAgent = isAgentProvider(graderTargetProvider) || graderTargetProvider.kind === "agentv";
-    evaluator = new LlmGraderEvaluator({
+    evaluator = new LlmGrader({
       resolveGraderProvider: async (evalContext) => {
         if (graderTargetProvider) return graderTargetProvider;
         if (evalContext.graderProvider) return evalContext.graderProvider;
@@ -15586,11 +15583,11 @@ var llmGraderFactory = (config, context) => {
         agentTimeoutMs
       );
       const isFromInlinePrompt = !c.resolvedPromptScript?.length && !c.resolvedPromptPath && !c.promptPath;
-      let evaluatorTemplateOverride;
+      let graderTemplateOverride;
       let evalCase = evalContext.evalCase;
       if (customPrompt) {
         if (!isFromInlinePrompt || containsTemplateVariables(customPrompt)) {
-          evaluatorTemplateOverride = customPrompt;
+          graderTemplateOverride = customPrompt;
         } else {
           evalCase = { ...evalCase, criteria: customPrompt };
         }
@@ -15598,7 +15595,7 @@ var llmGraderFactory = (config, context) => {
       return evaluator.evaluate({
         ...evalContext,
         evalCase,
-        evaluatorTemplateOverride,
+        graderTemplateOverride,
         evaluator: c
       });
     }
@@ -15606,7 +15603,7 @@ var llmGraderFactory = (config, context) => {
 };
 var codeFactory = (config, context) => {
   const c = config;
-  return new CodeEvaluator({
+  return new CodeGrader({
     command: c.command ?? c.script ?? [],
     cwd: c.resolvedCwd ?? c.cwd,
     agentTimeoutMs: context.agentTimeoutMs,
@@ -15617,19 +15614,19 @@ var codeFactory = (config, context) => {
 var compositeFactory = (config, context) => {
   const c = config;
   const evalFileDir = context.evalFileDir ?? process.cwd();
-  return new CompositeEvaluator({
+  return new CompositeGrader({
     config: c,
     cwd: evalFileDir,
     evaluatorFactory: {
       create: (memberConfig) => {
         const factory = context.registry.get(memberConfig.type);
         if (!factory) {
-          throw new Error(`Unsupported evaluator type in composite: ${memberConfig.type}`);
+          throw new Error(`Unsupported grader type in composite: ${memberConfig.type}`);
         }
         const result = factory(memberConfig, context);
         if (result instanceof Promise) {
           throw new Error(
-            `Evaluator factory for type "${memberConfig.type}" is async \u2014 not supported inside composite members. Use synchronous factories for composite child evaluators.`
+            `Grader factory for type "${memberConfig.type}" is async \u2014 not supported inside composite members. Use synchronous factories for composite child evaluators.`
           );
         }
         return result;
@@ -15638,35 +15635,35 @@ var compositeFactory = (config, context) => {
   });
 };
 var toolTrajectoryFactory = (config) => {
-  return new ToolTrajectoryEvaluator({
+  return new ToolTrajectoryGrader({
     config
   });
 };
 var fieldAccuracyFactory = (config) => {
-  return new FieldAccuracyEvaluator({
+  return new FieldAccuracyGrader({
     config
   });
 };
 var latencyFactory = (config) => {
-  return new LatencyEvaluator({ config });
+  return new LatencyGrader({ config });
 };
 var costFactory = (config) => {
-  return new CostEvaluator({ config });
+  return new CostGrader({ config });
 };
 var tokenUsageFactory = (config) => {
-  return new TokenUsageEvaluator({ config });
+  return new TokenUsageGrader({ config });
 };
 var executionMetricsFactory = (config) => {
-  return new ExecutionMetricsEvaluator({
+  return new ExecutionMetricsGrader({
     config
   });
 };
 var skillTriggerFactory = (config) => {
-  return new SkillTriggerEvaluator(config);
+  return new SkillTriggerGrader(config);
 };
 var containsFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("contains", (ctx) => {
+  return new DeterministicAssertionGrader("contains", (ctx) => {
     const result = runContainsAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -15678,7 +15675,7 @@ var containsFactory = (config) => {
 };
 var regexFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("regex", (ctx) => {
+  return new DeterministicAssertionGrader("regex", (ctx) => {
     const result = runRegexAssertion(ctx.candidate, c.value, c.flags);
     return {
       score: result.score,
@@ -15689,7 +15686,7 @@ var regexFactory = (config) => {
   });
 };
 var isJsonFactory = () => {
-  return new DeterministicAssertionEvaluator("is-json", (ctx) => {
+  return new DeterministicAssertionGrader("is-json", (ctx) => {
     const result = runIsJsonAssertion(ctx.candidate);
     return {
       score: result.score,
@@ -15701,7 +15698,7 @@ var isJsonFactory = () => {
 };
 var equalsFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("equals", (ctx) => {
+  return new DeterministicAssertionGrader("equals", (ctx) => {
     const result = runEqualsAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -15713,7 +15710,7 @@ var equalsFactory = (config) => {
 };
 var containsAnyFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("contains-any", (ctx) => {
+  return new DeterministicAssertionGrader("contains-any", (ctx) => {
     const result = runContainsAnyAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -15725,7 +15722,7 @@ var containsAnyFactory = (config) => {
 };
 var containsAllFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("contains-all", (ctx) => {
+  return new DeterministicAssertionGrader("contains-all", (ctx) => {
     const result = runContainsAllAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -15737,7 +15734,7 @@ var containsAllFactory = (config) => {
 };
 var icontainsFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("icontains", (ctx) => {
+  return new DeterministicAssertionGrader("icontains", (ctx) => {
     const result = runIcontainsAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -15749,7 +15746,7 @@ var icontainsFactory = (config) => {
 };
 var icontainsAnyFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("icontains-any", (ctx) => {
+  return new DeterministicAssertionGrader("icontains-any", (ctx) => {
     const result = runIcontainsAnyAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -15761,7 +15758,7 @@ var icontainsAnyFactory = (config) => {
 };
 var icontainsAllFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("icontains-all", (ctx) => {
+  return new DeterministicAssertionGrader("icontains-all", (ctx) => {
     const result = runIcontainsAllAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -15773,7 +15770,7 @@ var icontainsAllFactory = (config) => {
 };
 var startsWithFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("starts-with", (ctx) => {
+  return new DeterministicAssertionGrader("starts-with", (ctx) => {
     const result = runStartsWithAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -15785,7 +15782,7 @@ var startsWithFactory = (config) => {
 };
 var endsWithFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("ends-with", (ctx) => {
+  return new DeterministicAssertionGrader("ends-with", (ctx) => {
     const result = runEndsWithAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -15796,7 +15793,7 @@ var endsWithFactory = (config) => {
   });
 };
 function createBuiltinRegistry() {
-  const registry = new EvaluatorRegistry();
+  const registry = new GraderRegistry();
   registry.register("llm-grader", llmGraderFactory).register("code-grader", codeFactory).register("composite", compositeFactory).register("tool-trajectory", toolTrajectoryFactory).register("field-accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token-usage", tokenUsageFactory).register("execution-metrics", executionMetricsFactory).register("skill-trigger", skillTriggerFactory).register("contains", containsFactory).register("contains-any", containsAnyFactory).register("contains-all", containsAllFactory).register("icontains", icontainsFactory).register("icontains-any", icontainsAnyFactory).register("icontains-all", icontainsAllFactory).register("starts-with", startsWithFactory).register("ends-with", endsWithFactory).register("regex", regexFactory).register("is-json", isJsonFactory).register("equals", equalsFactory).register("inline-assert", (config) => {
     const fn = config[INLINE_ASSERT_FN];
     if (!fn) {
@@ -15804,7 +15801,7 @@ function createBuiltinRegistry() {
         `No inline assert function found on config for "${config.name}". Inline assert functions must be attached via INLINE_ASSERT_FN symbol.`
       );
     }
-    return new InlineAssertEvaluator(fn, config.name ?? "inline-assert");
+    return new InlineAssertGrader(fn, config.name ?? "inline-assert");
   });
   return registry;
 }
@@ -15841,7 +15838,7 @@ async function discoverAssertions(registry, baseDir) {
       continue;
     }
     const factory = (_config, context) => {
-      return new CodeEvaluator({
+      return new CodeGrader({
         command: ["bun", "run", filePath],
         agentTimeoutMs: context.agentTimeoutMs
       });
@@ -15885,7 +15882,7 @@ async function discoverGraders(registry, baseDir) {
       continue;
     }
     const factory = (_config, context) => {
-      return new CodeEvaluator({
+      return new CodeGrader({
         command: ["bun", "run", filePath],
         agentTimeoutMs: context.agentTimeoutMs
       });
@@ -16727,10 +16724,10 @@ function buildSkippedEvaluatorError(scores) {
   }
   const messages = skippedScores.map((score) => {
     const label = score.name || score.type;
-    const assertionMessage = score.assertions.find((assertion) => !assertion.passed)?.text ?? "Evaluator skipped";
+    const assertionMessage = score.assertions.find((assertion) => !assertion.passed)?.text ?? "Grader skipped";
     return `${label}: ${assertionMessage}`;
   });
-  return messages.length === 1 ? messages[0] : `Evaluators skipped: ${messages.join(" | ")}`;
+  return messages.length === 1 ? messages[0] : `Graders skipped: ${messages.join(" | ")}`;
 }
 function usesFileReferencePrompt(provider) {
   return isAgentProvider(provider) || provider.kind === "cli";
@@ -16899,7 +16896,7 @@ async function runEvaluation(options) {
     cleanupWorkspaces,
     trials,
     streamCallbacks,
-    totalBudgetUsd,
+    budgetUsd,
     failOnError,
     poolWorkspaces,
     poolMaxSlots: configPoolMaxSlots,
@@ -17428,7 +17425,7 @@ async function runEvaluation(options) {
     async function dispatchTest(evalCase, depResults) {
       const workerId = nextWorkerId++;
       workerIdByEvalId.set(evalCase.id, workerId);
-      if (totalBudgetUsd !== void 0 && budgetExhausted) {
+      if (budgetUsd !== void 0 && budgetExhausted) {
         const budgetResult = {
           timestamp: (now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
           testId: evalCase.id,
@@ -17438,13 +17435,13 @@ async function runEvaluation(options) {
           assertions: [],
           output: [],
           target: target.name,
-          error: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${totalBudgetUsd.toFixed(4)})`,
+          error: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${budgetUsd.toFixed(4)})`,
           budgetExceeded: true,
           executionStatus: "execution_error",
           failureStage: "setup",
           failureReasonCode: "budget_exceeded",
           executionError: {
-            message: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${totalBudgetUsd.toFixed(4)})`,
+            message: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${budgetUsd.toFixed(4)})`,
             stage: "setup"
           }
         };
@@ -17541,7 +17538,7 @@ async function runEvaluation(options) {
           ...depResults && Object.keys(depResults).length > 0 ? { dependencyResults: depResults } : {}
         };
         let result = trials && trials.count > 1 ? await runEvalCaseWithTrials(runCaseOptions, trials) : await runEvalCase(runCaseOptions);
-        if (totalBudgetUsd !== void 0) {
+        if (budgetUsd !== void 0) {
           let caseCost;
           if (result.trials && result.trials.length > 0) {
             const trialCostSum = result.trials.reduce((sum, t) => sum + (t.costUsd ?? 0), 0);
@@ -17553,7 +17550,7 @@ async function runEvaluation(options) {
           }
           if (caseCost !== void 0) {
             cumulativeBudgetCost += caseCost;
-            if (cumulativeBudgetCost >= totalBudgetUsd) {
+            if (cumulativeBudgetCost >= budgetUsd) {
               budgetExhausted = true;
             }
           }
@@ -18695,7 +18692,7 @@ async function evaluateCandidate(options) {
       };
     }
   }
-  const evaluatorRequest = scores ? void 0 : score.evaluatorRawRequest;
+  const evaluatorRequest = scores ? void 0 : score.graderRawRequest;
   const effectiveAgentRequest = agentRequest && Object.keys(agentRequest).length > 0 ? agentRequest : void 0;
   const requests = effectiveAgentRequest || lmRequest || evaluatorRequest ? {
     ...effectiveAgentRequest ? { agent: effectiveAgentRequest } : {},
@@ -18911,7 +18908,7 @@ async function runEvaluatorList(options) {
         weight,
         verdict: score2.verdict,
         assertions: score2.assertions,
-        input: score2.evaluatorRawRequest,
+        input: score2.graderRawRequest,
         target: score2.graderTarget,
         details: score2.details,
         scores: mapChildResults(score2.scores),
@@ -18927,7 +18924,7 @@ async function runEvaluatorList(options) {
         score: 0,
         verdict: "fail",
         assertions: [
-          { text: `Evaluator '${evaluatorConfig.name}' failed: ${message}`, passed: false }
+          { text: `Grader '${evaluatorConfig.name}' failed: ${message}`, passed: false }
         ],
         expectedAspectCount: 1
       };
@@ -18948,7 +18945,7 @@ async function runEvaluatorList(options) {
         verdict: "fail",
         assertions: [
           {
-            text: `Evaluator '${evaluatorConfig.name ?? "unknown"}' failed: ${message}`,
+            text: `Grader '${evaluatorConfig.name ?? "unknown"}' failed: ${message}`,
             passed: false
           }
         ],
@@ -19005,7 +19002,7 @@ function filterEvalCases(evalCases, filter) {
   return evalCases.filter((evalCase) => matchesFilter3(evalCase.id, filter));
 }
 function buildEvaluatorRegistry(overrides, resolveGraderProvider) {
-  const llmGrader = overrides?.["llm-grader"] ?? new LlmGraderEvaluator({
+  const llmGrader = overrides?.["llm-grader"] ?? new LlmGrader({
     resolveGraderProvider: async (context) => {
       if (context.graderProvider) {
         return context.graderProvider;
@@ -19496,7 +19493,7 @@ function mapChildResults(children) {
     weight: child.weight,
     verdict: child.verdict,
     assertions: child.assertions,
-    input: child.evaluatorRawRequest,
+    input: child.graderRawRequest,
     scores: mapChildResults(child.scores),
     details: child.details,
     tokenUsage: child.tokenUsage
@@ -21599,22 +21596,21 @@ function createAgentKernel() {
 }
 export {
   COMMON_TARGET_SETTINGS,
-  CodeEvaluator,
-  CompositeEvaluator,
-  CostEvaluator,
+  CodeGrader,
+  CompositeGrader,
+  CostGrader,
   DEFAULT_CATEGORY,
-  DEFAULT_EVALUATOR_TEMPLATE,
   DEFAULT_EVAL_PATTERNS,
   DEFAULT_EXPLORATION_TOOLS,
+  DEFAULT_GRADER_TEMPLATE,
   DEFAULT_THRESHOLD,
-  DeterministicAssertionEvaluator,
+  DeterministicAssertionGrader,
   DockerWorkspaceProvider,
-  EvaluatorRegistry,
-  ExecutionMetricsEvaluator,
-  FieldAccuracyEvaluator,
-  LatencyEvaluator,
-  LlmGraderEvaluator,
-  LlmGraderEvaluator as LlmJudgeEvaluator,
+  ExecutionMetricsGrader,
+  FieldAccuracyGrader,
+  GraderRegistry,
+  LatencyGrader,
+  LlmGrader,
   OTEL_BACKEND_PRESETS,
   OtelStreamingObserver,
   OtelTraceExporter,
@@ -21623,18 +21619,17 @@ export {
   ProviderRegistry,
   RepoManager,
   ResponseCache,
-  SkillTriggerEvaluator,
+  SkillTriggerGrader,
   TEST_MESSAGE_ROLES,
   TemplateNotDirectoryError,
   TemplateNotFoundError,
-  TokenUsageEvaluator,
-  ToolTrajectoryEvaluator,
+  TokenUsageGrader,
+  ToolTrajectoryGrader,
   TranscriptProvider,
   WorkspaceCreationError,
   WorkspacePoolManager,
   addBenchmark,
   assembleLlmGraderPrompt,
-  assembleLlmGraderPrompt as assembleLlmJudgePrompt,
   avgToolDurationMs,
   buildDirectoryChain,
   buildOutputSchema,
@@ -21674,7 +21669,6 @@ export {
   discoverCodexSessions,
   discoverCopilotSessions,
   discoverGraders,
-  discoverGraders as discoverJudges,
   discoverProviders,
   ensureResultsRepoClone,
   ensureVSCodeSubagents,
@@ -21716,7 +21710,7 @@ export {
   isAgentSkillsFormat,
   isContent,
   isContentArray,
-  isEvaluatorKind,
+  isGraderKind,
   isJsonObject,
   isJsonValue,
   isNonEmptyString,