@agentv/core 2.12.0 → 2.14.0-next.1

package/dist/index.js

@@ -17,7 +17,7 @@ import {
   readTextFile,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-7HPKTRFZ.js";
+} from "./chunk-N55K52OO.js";
 import {
   OtlpJsonFileExporter
 } from "./chunk-HFSYZHGF.js";
@@ -396,6 +396,11 @@ async function loadConfig(evalFilePath, repoRoot) {
         continue;
       }
       const config = parsed;
+      const requiredVersion = parsed.required_version;
+      if (requiredVersion !== void 0 && typeof requiredVersion !== "string") {
+        logWarning(`Invalid required_version in ${configPath}, expected string`);
+        continue;
+      }
       const guidelinePatterns = config.guideline_patterns;
       if (guidelinePatterns !== void 0 && !Array.isArray(guidelinePatterns)) {
         logWarning(`Invalid guideline_patterns in ${configPath}, expected array`);
@@ -419,6 +424,7 @@ async function loadConfig(evalFilePath, repoRoot) {
         configPath
       );
       return {
+        required_version: requiredVersion,
         guideline_patterns: guidelinePatterns,
         eval_patterns: evalPatterns,
         execution: executionDefaults
@@ -562,6 +568,22 @@ function extractTotalBudgetUsd(suite) {
   );
   return void 0;
 }
+function extractFailOnError(suite) {
+  const execution = suite.execution;
+  if (!execution || typeof execution !== "object" || Array.isArray(execution)) {
+    return void 0;
+  }
+  const executionObj = execution;
+  const raw = executionObj.fail_on_error ?? executionObj.failOnError;
+  if (raw === void 0 || raw === null) {
+    return void 0;
+  }
+  if (typeof raw === "boolean") {
+    return raw;
+  }
+  logWarning(`Invalid execution.fail_on_error: ${raw}. Must be true or false. Ignoring.`);
+  return void 0;
+}
 function parseExecutionDefaults(raw, configPath) {
   if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
     return void 0;
@@ -660,6 +682,9 @@ function validateTemplateVariables(content, source) {
 // src/evaluation/loaders/evaluator-parser.ts
 var ANSI_YELLOW4 = "\x1B[33m";
 var ANSI_RESET4 = "\x1B[0m";
+function normalizeEvaluatorType(type) {
+  return type.replace(/_/g, "-");
+}
 async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId) {
   const execution = rawEvalCase.execution;
   const executionObject = isJsonObject2(execution) ? execution : void 0;
@@ -690,7 +715,8 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       continue;
     }
     const rawName = asString(rawEvaluator.name);
-    const typeValue = rawEvaluator.type;
+    const rawType = rawEvaluator.type;
+    const typeValue = typeof rawType === "string" ? normalizeEvaluatorType(rawType) : rawType;
     const isCustomType = typeof typeValue === "string" && !isEvaluatorKind(typeValue);
     if (typeof typeValue !== "string") {
       logWarning2(`Skipping evaluator with invalid type in '${evalId}'`);
@@ -723,25 +749,25 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "code_judge") {
+    if (typeValue === "code-judge") {
       let command;
       const rawCommand = rawEvaluator.command ?? rawEvaluator.script;
       if (typeof rawCommand === "string") {
         const trimmed = rawCommand.trim();
         if (trimmed.length === 0) {
           throw new Error(
-            `Invalid code_judge command for evaluator '${name}' in '${evalId}': command cannot be empty`
+            `Invalid code-judge command for evaluator '${name}' in '${evalId}': command cannot be empty`
           );
         }
         command = parseCommandToArgv(trimmed);
       } else {
         command = asStringArray(
           rawCommand,
-          `code_judge command for evaluator '${name}' in '${evalId}'`
+          `code-judge command for evaluator '${name}' in '${evalId}'`
         );
       }
       if (!command) {
-        logWarning2(`Skipping code_judge evaluator '${name}' in '${evalId}': missing command`);
+        logWarning2(`Skipping code-judge evaluator '${name}' in '${evalId}': missing command`);
         continue;
       }
       const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
@@ -802,7 +828,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       }
       evaluators.push({
         name,
-        type: "code",
+        type: "code-judge",
         command,
         cwd,
         resolvedCwd,
@@ -828,7 +854,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
         continue;
       }
       const aggregatorType = asString(rawAggregator.type);
-      if (aggregatorType !== "weighted_average" && aggregatorType !== "code_judge" && aggregatorType !== "llm_judge" && aggregatorType !== "threshold") {
+      if (aggregatorType !== "weighted_average" && aggregatorType !== "code-judge" && aggregatorType !== "llm-judge" && aggregatorType !== "threshold") {
         logWarning2(
           `Skipping composite evaluator '${name}' in '${evalId}': invalid aggregator type '${aggregatorType}'`
         );
@@ -877,16 +903,16 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
           type: "weighted_average",
           ...Object.keys(parsedWeights).length > 0 ? { weights: parsedWeights } : {}
         };
-      } else if (aggregatorType === "code_judge") {
+      } else if (aggregatorType === "code-judge") {
         const aggregatorPath = asString(rawAggregator.path);
         if (!aggregatorPath) {
           logWarning2(
-            `Skipping composite evaluator '${name}' in '${evalId}': code_judge aggregator missing path`
+            `Skipping composite evaluator '${name}' in '${evalId}': code-judge aggregator missing path`
           );
           continue;
         }
         aggregator = {
-          type: "code_judge",
+          type: "code-judge",
           path: aggregatorPath,
           cwd: searchRoots[0]
         };
@@ -912,7 +938,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
           }
         }
         aggregator = {
-          type: "llm_judge",
+          type: "llm-judge",
           ...aggregatorPrompt ? { prompt: aggregatorPrompt } : {},
           ...promptPath2 ? { promptPath: promptPath2 } : {}
         };
@@ -930,11 +956,11 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "tool_trajectory") {
+    if (typeValue === "tool-trajectory") {
       const mode = asString(rawEvaluator.mode);
       if (mode !== "any_order" && mode !== "in_order" && mode !== "exact" && mode !== "subset" && mode !== "superset") {
         logWarning2(
-          `Skipping tool_trajectory evaluator '${name}' in '${evalId}': invalid mode '${mode}' (must be any_order, in_order, exact, subset, or superset)`
+          `Skipping tool-trajectory evaluator '${name}' in '${evalId}': invalid mode '${mode}' (must be any_order, in_order, exact, subset, or superset)`
         );
         continue;
       }
@@ -943,7 +969,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       if (rawMinimums !== void 0) {
         if (!isJsonObject2(rawMinimums)) {
           logWarning2(
-            `Skipping tool_trajectory evaluator '${name}' in '${evalId}': minimums must be an object`
+            `Skipping tool-trajectory evaluator '${name}' in '${evalId}': minimums must be an object`
           );
           continue;
         }
@@ -969,7 +995,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
             argsMatch2 = rawArgsMatch;
           } else {
             logWarning2(
-              `Invalid args_match '${rawArgsMatch}' for tool_trajectory evaluator '${name}' in '${evalId}': must be exact, superset, subset, ignore, or a string array`
+              `Invalid args_match '${rawArgsMatch}' for tool-trajectory evaluator '${name}' in '${evalId}': must be exact, superset, subset, ignore, or a string array`
             );
           }
         }
@@ -979,7 +1005,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       if (rawExpected !== void 0) {
         if (!Array.isArray(rawExpected)) {
           logWarning2(
-            `Skipping tool_trajectory evaluator '${name}' in '${evalId}': expected must be an array`
+            `Skipping tool-trajectory evaluator '${name}' in '${evalId}': expected must be an array`
           );
           continue;
         }
@@ -1025,13 +1051,13 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       }
       if (mode === "any_order" && !minimums) {
         logWarning2(
-          `Skipping tool_trajectory evaluator '${name}' in '${evalId}': any_order mode requires minimums`
+          `Skipping tool-trajectory evaluator '${name}' in '${evalId}': any_order mode requires minimums`
         );
         continue;
       }
       if ((mode === "in_order" || mode === "exact" || mode === "subset" || mode === "superset") && !expected) {
         logWarning2(
-          `Skipping tool_trajectory evaluator '${name}' in '${evalId}': ${mode} mode requires expected`
+          `Skipping tool-trajectory evaluator '${name}' in '${evalId}': ${mode} mode requires expected`
         );
         continue;
       }
@@ -1039,7 +1065,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required2 = parseRequired(rawEvaluator.required);
       const config2 = {
         name,
-        type: "tool_trajectory",
+        type: "tool-trajectory",
         mode,
         ...minimums ? { minimums } : {},
         ...expected ? { expected } : {},
@@ -1051,17 +1077,17 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       evaluators.push(config2);
       continue;
     }
-    if (typeValue === "field_accuracy") {
+    if (typeValue === "field-accuracy") {
       const rawFields = rawEvaluator.fields;
       if (!Array.isArray(rawFields)) {
         logWarning2(
-          `Skipping field_accuracy evaluator '${name}' in '${evalId}': missing fields array`
+          `Skipping field-accuracy evaluator '${name}' in '${evalId}': missing fields array`
         );
         continue;
       }
       if (rawFields.length === 0) {
         logWarning2(
-          `Skipping field_accuracy evaluator '${name}' in '${evalId}': fields array is empty`
+          `Skipping field-accuracy evaluator '${name}' in '${evalId}': fields array is empty`
         );
         continue;
       }
@@ -1069,7 +1095,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       for (const rawField of rawFields) {
         if (!isJsonObject2(rawField)) {
           logWarning2(
-            `Skipping invalid field entry in field_accuracy evaluator '${name}' (expected object)`
+            `Skipping invalid field entry in field-accuracy evaluator '${name}' (expected object)`
           );
           continue;
         }
@@ -1077,13 +1103,13 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
         const match = asString(rawField.match);
         if (!fieldPath) {
           logWarning2(
-            `Skipping field without path in field_accuracy evaluator '${name}' in '${evalId}'`
+            `Skipping field without path in field-accuracy evaluator '${name}' in '${evalId}'`
           );
           continue;
         }
         if (!match || !isValidFieldMatchType(match)) {
           logWarning2(
-            `Skipping field '${fieldPath}' with invalid match type '${match}' in evaluator '${name}' (must be exact, numeric_tolerance, or date). For fuzzy matching, use a code_judge evaluator.`
+            `Skipping field '${fieldPath}' with invalid match type '${match}' in evaluator '${name}' (must be exact, numeric_tolerance, or date). For fuzzy matching, use a code-judge evaluator.`
           );
           continue;
         }
@@ -1100,7 +1126,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       }
       if (fields.length === 0) {
         logWarning2(
-          `Skipping field_accuracy evaluator '${name}' in '${evalId}': no valid fields found`
+          `Skipping field-accuracy evaluator '${name}' in '${evalId}': no valid fields found`
         );
         continue;
       }
@@ -1110,7 +1136,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required2 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name,
-        type: "field_accuracy",
+        type: "field-accuracy",
         fields,
         ...validAggregation ? { aggregation: validAggregation } : {},
         ...weight2 !== void 0 ? { weight: weight2 } : {},
@@ -1159,7 +1185,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "token_usage") {
+    if (typeValue === "token-usage") {
       const maxTotal = rawEvaluator.max_total ?? rawEvaluator.maxTotal;
       const maxInput = rawEvaluator.max_input ?? rawEvaluator.maxInput;
       const maxOutput = rawEvaluator.max_output ?? rawEvaluator.maxOutput;
@@ -1173,7 +1199,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
         if (raw === void 0) continue;
         if (typeof raw !== "number" || !Number.isFinite(raw) || raw < 0) {
           logWarning2(
-            `Skipping token_usage evaluator '${name}' in '${evalId}': ${key} must be a non-negative finite number`
+            `Skipping token-usage evaluator '${name}' in '${evalId}': ${key} must be a non-negative finite number`
           );
           continue;
         }
@@ -1181,7 +1207,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       }
       if (validLimits.max_total === void 0 && validLimits.max_input === void 0 && validLimits.max_output === void 0) {
         logWarning2(
-          `Skipping token_usage evaluator '${name}' in '${evalId}': must set at least one of max_total, max_input, max_output`
+          `Skipping token-usage evaluator '${name}' in '${evalId}': must set at least one of max_total, max_input, max_output`
         );
         continue;
       }
@@ -1189,7 +1215,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required2 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name,
-        type: "token_usage",
+        type: "token-usage",
         ...validLimits,
         ...weight2 !== void 0 ? { weight: weight2 } : {},
         ...required2 !== void 0 ? { required: required2 } : {},
@@ -1197,7 +1223,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "execution_metrics") {
+    if (typeValue === "execution-metrics") {
       const maxToolCalls = rawEvaluator.max_tool_calls ?? rawEvaluator.maxToolCalls;
       const maxLlmCalls = rawEvaluator.max_llm_calls ?? rawEvaluator.maxLlmCalls;
       const maxTokens = rawEvaluator.max_tokens ?? rawEvaluator.maxTokens;
@@ -1220,7 +1246,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
         if (raw === void 0) continue;
         if (typeof raw !== "number" || !Number.isFinite(raw) || raw < 0) {
           logWarning2(
-            `Skipping execution_metrics evaluator '${name}' in '${evalId}': ${key} must be a non-negative finite number`
+            `Skipping execution-metrics evaluator '${name}' in '${evalId}': ${key} must be a non-negative finite number`
           );
           hasError = true;
           break;
@@ -1233,7 +1259,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const hasThreshold = validThresholds.max_tool_calls !== void 0 || validThresholds.max_llm_calls !== void 0 || validThresholds.max_tokens !== void 0 || validThresholds.max_cost_usd !== void 0 || validThresholds.max_duration_ms !== void 0 || validThresholds.target_exploration_ratio !== void 0;
       if (!hasThreshold) {
         logWarning2(
-          `Skipping execution_metrics evaluator '${name}' in '${evalId}': must set at least one threshold (max_tool_calls, max_llm_calls, max_tokens, max_cost_usd, max_duration_ms, or target_exploration_ratio)`
+          `Skipping execution-metrics evaluator '${name}' in '${evalId}': must set at least one threshold (max_tool_calls, max_llm_calls, max_tokens, max_cost_usd, max_duration_ms, or target_exploration_ratio)`
         );
         continue;
       }
@@ -1241,7 +1267,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required2 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name,
-        type: "execution_metrics",
+        type: "execution-metrics",
         ...validThresholds,
         ...weight2 !== void 0 ? { weight: weight2 } : {},
         ...required2 !== void 0 ? { required: required2 } : {},
@@ -1249,13 +1275,13 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "agent_judge") {
+    if (typeValue === "agent-judge") {
       const rawMaxSteps = rawEvaluator.max_steps ?? rawEvaluator.maxSteps;
       let maxSteps;
       if (rawMaxSteps !== void 0) {
         if (typeof rawMaxSteps !== "number" || !Number.isInteger(rawMaxSteps) || rawMaxSteps < 1 || rawMaxSteps > 50) {
           logWarning2(
-            `Skipping agent_judge evaluator '${name}' in '${evalId}': max_steps must be an integer 1-50`
+            `Skipping agent-judge evaluator '${name}' in '${evalId}': max_steps must be an integer 1-50`
           );
           continue;
         }
@@ -1266,7 +1292,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       if (rawTemperature !== void 0) {
         if (typeof rawTemperature !== "number" || rawTemperature < 0 || rawTemperature > 2) {
           logWarning2(
-            `Skipping agent_judge evaluator '${name}' in '${evalId}': temperature must be a number 0-2`
+            `Skipping agent-judge evaluator '${name}' in '${evalId}': temperature must be a number 0-2`
           );
           continue;
         }
@@ -1289,7 +1315,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required2 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name,
-        type: "agent_judge",
+        type: "agent-judge",
         ...agentPrompt ? { prompt: agentPrompt } : {},
         ...agentPromptPath ? { promptPath: agentPromptPath, resolvedPromptPath: agentPromptPath } : {},
         ...agentParsedRubrics && agentParsedRubrics.length > 0 ? { rubrics: agentParsedRubrics } : {},
@@ -1320,7 +1346,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "contains_any" || typeValue === "contains_all") {
+    if (typeValue === "contains-any" || typeValue === "contains-all") {
       const value = asStringArrayStrict(rawEvaluator.value);
       if (!value || value.length === 0) {
         logWarning2(
@@ -1358,7 +1384,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "icontains_any" || typeValue === "icontains_all") {
+    if (typeValue === "icontains-any" || typeValue === "icontains-all") {
       const value = asStringArrayStrict(rawEvaluator.value);
       if (!value || value.length === 0) {
         logWarning2(
@@ -1378,7 +1404,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "starts_with" || typeValue === "ends_with") {
+    if (typeValue === "starts-with" || typeValue === "ends-with") {
       const value = asString(rawEvaluator.value);
       if (!value) {
         logWarning2(`Skipping ${typeValue} evaluator '${name}' in '${evalId}': missing value`);
@@ -1416,12 +1442,12 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "is_json") {
+    if (typeValue === "is-json") {
       const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
       const required2 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name,
-        type: "is_json",
+        type: "is-json",
         ...weight2 !== void 0 ? { weight: weight2 } : {},
         ...required2 !== void 0 ? { required: required2 } : {},
         ...negate !== void 0 ? { negate } : {}
@@ -1469,7 +1495,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required2 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name,
-        type: "llm_judge",
+        type: "llm-judge",
         rubrics: parsedCriteria,
         ...weight2 !== void 0 ? { weight: weight2 } : {},
         ...required2 !== void 0 ? { required: required2 } : {},
@@ -1536,7 +1562,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required2 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name,
-        type: "llm_judge",
+        type: "llm-judge",
         rubrics: parsedRubrics,
         ...weight2 !== void 0 ? { weight: weight2 } : {},
         ...required2 !== void 0 ? { required: required2 } : {},
@@ -1568,7 +1594,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
     const finalConfig = promptScriptConfig ?? (Object.keys(mergedConfig).length > 0 ? mergedConfig : void 0);
     evaluators.push({
       name,
-      type: "llm_judge",
+      type: "llm-judge",
       prompt,
       promptPath,
       ...promptPath ? { resolvedPromptPath: promptPath } : {},
@@ -1584,15 +1610,15 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
 }
 var ASSERTION_TYPES = /* @__PURE__ */ new Set([
   "contains",
-  "contains_any",
-  "contains_all",
+  "contains-any",
+  "contains-all",
   "icontains",
-  "icontains_any",
-  "icontains_all",
-  "starts_with",
-  "ends_with",
+  "icontains-any",
+  "icontains-all",
+  "starts-with",
+  "ends-with",
   "regex",
-  "is_json",
+  "is-json",
   "equals",
   "rubrics"
 ]);
@@ -1605,24 +1631,24 @@ function generateAssertionName(typeValue, rawEvaluator) {
   switch (typeValue) {
     case "contains":
       return value ? `contains-${value}` : "contains";
-    case "contains_any":
-      return arrayValue ? `contains_any-${arrayValue.length}` : "contains_any";
-    case "contains_all":
-      return arrayValue ? `contains_all-${arrayValue.length}` : "contains_all";
+    case "contains-any":
+      return arrayValue ? `contains-any-${arrayValue.length}` : "contains-any";
+    case "contains-all":
+      return arrayValue ? `contains-all-${arrayValue.length}` : "contains-all";
     case "icontains":
       return value ? `icontains-${value}` : "icontains";
-    case "icontains_any":
-      return arrayValue ? `icontains_any-${arrayValue.length}` : "icontains_any";
-    case "icontains_all":
-      return arrayValue ? `icontains_all-${arrayValue.length}` : "icontains_all";
-    case "starts_with":
-      return value ? `starts_with-${value}` : "starts_with";
-    case "ends_with":
-      return value ? `ends_with-${value}` : "ends_with";
+    case "icontains-any":
+      return arrayValue ? `icontains-any-${arrayValue.length}` : "icontains-any";
+    case "icontains-all":
+      return arrayValue ? `icontains-all-${arrayValue.length}` : "icontains-all";
+    case "starts-with":
+      return value ? `starts-with-${value}` : "starts-with";
+    case "ends-with":
+      return value ? `ends-with-${value}` : "ends-with";
     case "regex":
       return value ? `regex-${value.length > 30 ? value.slice(0, 30) : value}` : "regex";
-    case "is_json":
-      return "is_json";
+    case "is-json":
+      return "is-json";
     case "equals":
       return value ? `equals-${value}` : "equals";
     case "rubrics":
@@ -1635,8 +1661,9 @@ function coerceEvaluator(candidate, contextId) {
   if (typeof candidate !== "string") {
     return void 0;
   }
-  if (isEvaluatorKind(candidate)) {
-    return candidate;
+  const normalized = normalizeEvaluatorType(candidate);
+  if (isEvaluatorKind(normalized)) {
+    return normalized;
   }
   logWarning2(`Unknown evaluator '${candidate}' in ${contextId}, falling back to default`);
   return void 0;
@@ -1682,6 +1709,16 @@ function parseCommandToArgv(command) {
 function isJsonObject2(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
+var CRITERIA_CONSUMER_TYPES = /* @__PURE__ */ new Set(["llm-judge", "agent-judge", "code-judge"]);
+function warnUnconsumedCriteria(criteria, evaluators, testId) {
+  if (!criteria?.trim() || !evaluators || evaluators.length === 0) return;
+  const hasConsumer = evaluators.some((e) => CRITERIA_CONSUMER_TYPES.has(e.type));
+  if (!hasConsumer) {
+    logWarning2(
+      `Test '${testId}': criteria is defined but no evaluator in assert will evaluate it. Add 'type: llm-judge' to assert, or remove criteria if it is documentation-only.`
+    );
+  }
+}
 function logWarning2(message, details) {
   if (details && details.length > 0) {
     const detailBlock = details.join("\n");
@@ -1931,7 +1968,7 @@ function parseInlineRubrics(rawRubrics) {
   }
   return {
     name: "rubric",
-    type: "llm_judge",
+    type: "llm-judge",
     rubrics: rubricItems
   };
 }
@@ -2316,7 +2353,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
   const rawCases = parseJsonlContent(rawFile, evalFilePath);
   const fallbackDataset = path6.basename(absoluteTestPath, ".jsonl") || "eval";
   const datasetName = sidecar.dataset && sidecar.dataset.trim().length > 0 ? sidecar.dataset : fallbackDataset;
-  const globalEvaluator = coerceEvaluator(sidecar.evaluator, "sidecar") ?? "llm_judge";
+  const globalEvaluator = coerceEvaluator(sidecar.evaluator, "sidecar") ?? "llm-judge";
   const globalExecution = sidecar.execution;
   if (verbose) {
     console.log(`
@@ -2404,6 +2441,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
         evaluators = evaluators ? [rubricEvaluator, ...evaluators] : [rubricEvaluator];
       }
     }
+    warnUnconsumedCriteria(outcome, evaluators, id ?? "unknown");
     const userFilePaths = [];
     for (const segment of inputSegments) {
       if (segment.type === "file" && typeof segment.resolvedPath === "string") {
@@ -2757,13 +2795,15 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
   }
   const { tests, parsed } = await loadTestsFromYaml(evalFilePath, repoRoot, options);
   const metadata = parseMetadata(parsed);
+  const failOnError = extractFailOnError(parsed);
   return {
     tests,
     trials: extractTrialsConfig(parsed),
     targets: extractTargetsFromSuite(parsed),
     cacheConfig: extractCacheConfig(parsed),
     totalBudgetUsd: extractTotalBudgetUsd(parsed),
-    ...metadata !== void 0 && { metadata }
+    ...metadata !== void 0 && { metadata },
+    ...failOnError !== void 0 && { failOnError }
   };
 }
 var loadEvalSuite = loadTestSuite;
@@ -2794,7 +2834,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
   const fallbackDataset = path8.basename(absoluteTestPath).replace(/\.ya?ml$/i, "") || "eval";
   const datasetName = datasetNameFromSuite && datasetNameFromSuite.length > 0 ? datasetNameFromSuite : fallbackDataset;
   const rawTestcases = resolveTests(suite);
-  const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm_judge";
+  const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm-judge";
   const evalFileDir = path8.dirname(absoluteTestPath);
   let expandedTestcases;
   if (typeof rawTestcases === "string") {
@@ -2891,6 +2931,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
         evaluators = evaluators ? [rubricEvaluator, ...evaluators] : [rubricEvaluator];
       }
     }
+    warnUnconsumedCriteria(outcome, evaluators, id ?? "unknown");
     const userFilePaths = [];
     for (const segment of inputSegments) {
       if (segment.type === "file" && typeof segment.resolvedPath === "string") {
@@ -8871,7 +8912,7 @@ function toCamelCaseDeep(obj) {
 // src/evaluation/evaluators/code-evaluator.ts
 var FILE_BACKED_OUTPUT_THRESHOLD = 5e4;
 var CodeEvaluator = class {
-  kind = "code";
+  kind = "code-judge";
   command;
   cwd;
   agentTimeoutMs;
@@ -9079,7 +9120,7 @@ var scoreRangeEvaluationSchema = z3.object({
   overall_reasoning: z3.string().describe("Overall assessment summary (1-2 sentences)").optional()
 });
 var LlmJudgeEvaluator = class {
-  kind = "llm_judge";
+  kind = "llm-judge";
   resolveJudgeProvider;
   maxOutputTokens;
   temperature;
@@ -9096,7 +9137,7 @@ var LlmJudgeEvaluator = class {
       throw new Error("No judge provider available for LLM grading");
     }
     const config = context.evaluator;
-    if (config?.type === "llm_judge" && config.rubrics && config.rubrics.length > 0) {
+    if (config?.type === "llm-judge" && config.rubrics && config.rubrics.length > 0) {
       return this.evaluateWithRubrics(context, judgeProvider, config.rubrics);
     }
     return this.evaluateFreeform(context, judgeProvider);
@@ -9170,7 +9211,7 @@ ${context.fileChanges}`;
   async evaluateWithRubrics(context, judgeProvider, rubrics) {
     if (!rubrics || rubrics.length === 0) {
       throw new Error(
-        `No rubrics found for evaluator "${context.evaluator?.name ?? "llm_judge"}". Run "agentv generate rubrics" first.`
+        `No rubrics found for evaluator "${context.evaluator?.name ?? "llm-judge"}". Run "agentv generate rubrics" first.`
       );
     }
     const hasScoreRanges = rubrics.some((r) => r.score_ranges && r.score_ranges.length > 0);
@@ -9506,9 +9547,9 @@ var CompositeEvaluator = class {
   async aggregate(results, context) {
     const aggregator = this.config.aggregator;
     switch (aggregator.type) {
-      case "code_judge":
+      case "code-judge":
         return this.runCodeAggregator(results, aggregator.path, aggregator.cwd ?? this.cwd);
-      case "llm_judge":
+      case "llm-judge":
         return this.runLlmAggregator(results, context, aggregator);
       case "threshold":
         return this.runThreshold(results, aggregator.threshold);
@@ -9651,7 +9692,7 @@ var CompositeEvaluator = class {
         expectedAspectCount: hits.length + misses.length || 1,
         reasoning,
         evaluatorRawRequest: {
-          aggregator: "code_judge",
+          aggregator: "code-judge",
           script: scriptPath
         },
         scores
@@ -9666,7 +9707,7 @@ var CompositeEvaluator = class {
         expectedAspectCount: 1,
         reasoning: message,
         evaluatorRawRequest: {
-          aggregator: "code_judge",
+          aggregator: "code-judge",
           script: scriptPath,
           error: message
         },
@@ -9697,7 +9738,7 @@ var CompositeEvaluator = class {
     const userPrompt = promptTemplate.replace(/\{\{EVALUATOR_RESULTS_JSON\}\}/g, resultsJson);
     const systemPrompt = buildOutputSchema();
     const evaluatorRawRequest = {
-      aggregator: "llm_judge",
+      aggregator: "llm-judge",
       userPrompt,
       systemPrompt,
       target: judgeProvider.targetName
@@ -9809,7 +9850,7 @@ var CostEvaluator = class {
 
 // src/evaluation/evaluators/execution-metrics.ts
 var ExecutionMetricsEvaluator = class {
-  kind = "execution_metrics";
+  kind = "execution-metrics";
   config;
   constructor(options) {
     this.config = options.config;
@@ -9835,7 +9876,7 @@ var ExecutionMetricsEvaluator = class {
         expectedAspectCount: 1,
         reasoning: "Execution metrics not available - no trace summary provided",
         evaluatorRawRequest: {
-          type: "execution_metrics",
+          type: "execution-metrics",
           config: this.extractConfiguredThresholds(),
           actual: null
         }
@@ -9944,7 +9985,7 @@ var ExecutionMetricsEvaluator = class {
     if (actualMetrics.exploration_ratio !== void 0) {
       reasoningParts.push(`exploration_ratio=${actualMetrics.exploration_ratio.toFixed(2)}`);
     }
-    const reasoning = reasoningParts.length > 0 ? `execution_metrics ${reasoningParts.join(", ")}` : "No metrics evaluated";
+    const reasoning = reasoningParts.length > 0 ? `execution-metrics ${reasoningParts.join(", ")}` : "No metrics evaluated";
     return {
       score,
       verdict: scoreToVerdict(score),
@@ -9953,7 +9994,7 @@ var ExecutionMetricsEvaluator = class {
       expectedAspectCount: totalChecks || 1,
       reasoning,
       evaluatorRawRequest: {
-        type: "execution_metrics",
+        type: "execution-metrics",
         config: this.extractConfiguredThresholds(),
         actual: this.filterDefinedMetrics(actualMetrics)
       }
@@ -10041,7 +10082,7 @@ var MONTH_NAMES = {
   december: 11
 };
 var FieldAccuracyEvaluator = class {
-  kind = "field_accuracy";
+  kind = "field-accuracy";
   config;
   constructor(options) {
     this.config = options.config;
@@ -10495,7 +10536,7 @@ var BINARY_EXTENSIONS = /* @__PURE__ */ new Set([
   ".dylib"
 ]);
 var AgentJudgeEvaluator = class {
-  kind = "agent_judge";
+  kind = "agent-judge";
   resolveJudgeProvider;
   maxSteps;
   temperature;
@@ -10520,24 +10561,24 @@ var AgentJudgeEvaluator = class {
   async evaluateBuiltIn(context) {
     const judgeProvider = await this.resolveJudgeProvider(context);
     if (!judgeProvider) {
-      throw new Error("No judge provider available for agent_judge evaluation");
+      throw new Error("No judge provider available for agent-judge evaluation");
     }
     const model = judgeProvider.asLanguageModel?.();
     if (!model) {
       throw new Error(
-        `Judge provider '${judgeProvider.targetName}' does not support asLanguageModel() \u2014 required for built-in agent_judge mode`
+        `Judge provider '${judgeProvider.targetName}' does not support asLanguageModel() \u2014 required for built-in agent-judge mode`
       );
     }
     const workspacePath = context.workspacePath;
     if (!workspacePath) {
       throw new Error(
-        "agent_judge evaluator requires a workspace_template target (workspacePath is not set)"
+        "agent-judge evaluator requires a workspace_template target (workspacePath is not set)"
       );
     }
     const systemPrompt = this.buildSystemPrompt(context);
     const userPrompt = this.buildUserPrompt(context);
     const config = context.evaluator;
-    const rubrics = config?.type === "agent_judge" ? config.rubrics : void 0;
+    const rubrics = config?.type === "agent-judge" ? config.rubrics : void 0;
     const fsTools = createFilesystemTools(workspacePath);
     const evaluatorRawRequest = {
       mode: "built-in",
@@ -10568,7 +10609,7 @@ var AgentJudgeEvaluator = class {
         score: 0,
         verdict: "fail",
         hits: [],
-        misses: [`agent_judge built-in evaluation failed: ${message}`],
+        misses: [`agent-judge built-in evaluation failed: ${message}`],
         expectedAspectCount: 1,
         evaluatorRawRequest,
         details: { mode: "built-in", error: message }
@@ -10600,14 +10641,14 @@ var AgentJudgeEvaluator = class {
           score: 0,
           verdict: "fail",
           hits: [],
-          misses: ["agent_judge judge_target returned no assistant response"],
+          misses: ["agent-judge judge_target returned no assistant response"],
           expectedAspectCount: 1,
           evaluatorRawRequest,
           details: { mode: "judge_target", judge_target: provider.targetName }
         };
       }
       const config = context.evaluator;
-      const rubrics = config?.type === "agent_judge" ? config.rubrics : void 0;
+      const rubrics = config?.type === "agent-judge" ? config.rubrics : void 0;
       const details = {
         mode: "judge_target",
         judge_target: provider.targetName
@@ -10619,7 +10660,7 @@ var AgentJudgeEvaluator = class {
         score: 0,
         verdict: "fail",
         hits: [],
-        misses: [`agent_judge judge_target evaluation failed: ${message}`],
+        misses: [`agent-judge judge_target evaluation failed: ${message}`],
         expectedAspectCount: 1,
         evaluatorRawRequest,
         details: {
@@ -10670,7 +10711,7 @@ var AgentJudgeEvaluator = class {
         score: 0,
         verdict: "fail",
         hits: [],
-        misses: ["Failed to parse agent_judge response as valid evaluation JSON"],
+        misses: ["Failed to parse agent-judge response as valid evaluation JSON"],
         expectedAspectCount: 1,
         evaluatorRawRequest,
         details
@@ -10683,7 +10724,7 @@ var AgentJudgeEvaluator = class {
    */
   buildSystemPrompt(context) {
     const config = context.evaluator;
-    const rubrics = config?.type === "agent_judge" ? config.rubrics : void 0;
+    const rubrics = config?.type === "agent-judge" ? config.rubrics : void 0;
     const parts = [
       "You are an expert evaluator with access to the workspace filesystem.",
       "Use the provided tools to investigate the workspace and verify the criteria are met.",
@@ -10714,7 +10755,7 @@ var AgentJudgeEvaluator = class {
       return substituteVariables(this.evaluatorTemplate, variables);
     }
     const config = context.evaluator;
-    const rubrics = config?.type === "agent_judge" ? config.rubrics : void 0;
+    const rubrics = config?.type === "agent-judge" ? config.rubrics : void 0;
     const parts = [
       "Evaluate the candidate answer by investigating the workspace.",
       "",
@@ -10757,7 +10798,7 @@ var AgentJudgeEvaluator = class {
   buildDelegatedPrompt(context) {
     const formattedQuestion = context.promptInputs.question && context.promptInputs.question.trim().length > 0 ? context.promptInputs.question : context.evalCase.question;
     const config = context.evaluator;
-    const rubrics = config?.type === "agent_judge" ? config.rubrics : void 0;
+    const rubrics = config?.type === "agent-judge" ? config.rubrics : void 0;
     if (this.evaluatorTemplate) {
       const variables = {
         [TEMPLATE_VARIABLES.ANSWER]: context.candidate.trim(),
@@ -10839,11 +10880,11 @@ function createFilesystemTools(workspacePath) {
       execute: async (input) => {
         try {
           const resolved = resolveSandboxed(workspacePath, input.path);
-          const stat7 = await fs2.stat(resolved);
-          if (stat7.isDirectory()) {
+          const stat8 = await fs2.stat(resolved);
+          if (stat8.isDirectory()) {
             return { error: `'${input.path}' is a directory, not a file` };
           }
-          const buffer = Buffer.alloc(Math.min(stat7.size, MAX_FILE_SIZE));
+          const buffer = Buffer.alloc(Math.min(stat8.size, MAX_FILE_SIZE));
           const fd = await fs2.open(resolved, "r");
           try {
             await fd.read(buffer, 0, buffer.length, 0);
@@ -10851,8 +10892,8 @@ function createFilesystemTools(workspacePath) {
             await fd.close();
           }
           const content = buffer.toString("utf-8");
-          const truncated = stat7.size > MAX_FILE_SIZE;
-          return { content, truncated, size: stat7.size };
+          const truncated = stat8.size > MAX_FILE_SIZE;
+          return { content, truncated, size: stat8.size };
         } catch (error) {
           return { error: error instanceof Error ? error.message : String(error) };
         }
@@ -10896,8 +10937,8 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
       const ext = path30.extname(entry.name).toLowerCase();
       if (BINARY_EXTENSIONS.has(ext)) continue;
       try {
-        const stat7 = await fs2.stat(fullPath);
-        if (stat7.size > MAX_FILE_SIZE) continue;
+        const stat8 = await fs2.stat(fullPath);
+        if (stat8.size > MAX_FILE_SIZE) continue;
         const content = await fs2.readFile(fullPath, "utf-8");
         const lines = content.split("\n");
         for (let i = 0; i < lines.length; i++) {
@@ -11059,7 +11100,7 @@ function assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChan
 
 // src/evaluation/evaluators/token-usage.ts
 var TokenUsageEvaluator = class {
-  kind = "token_usage";
+  kind = "token-usage";
   config;
   constructor(options) {
     this.config = options.config;
@@ -11082,7 +11123,7 @@ var TokenUsageEvaluator = class {
         expectedAspectCount,
         reasoning: "Token usage not reported by provider",
         evaluatorRawRequest: {
-          type: "token_usage",
+          type: "token-usage",
           max_total: maxTotal ?? null,
           max_input: maxInput ?? null,
           max_output: maxOutput ?? null,
@@ -11124,9 +11165,9 @@ var TokenUsageEvaluator = class {
       hits,
       misses,
       expectedAspectCount,
-      reasoning: `token_usage input=${input}, output=${output}, cached=${cached}, total=${total}`,
+      reasoning: `token-usage input=${input}, output=${output}, cached=${cached}, total=${total}`,
       evaluatorRawRequest: {
-        type: "token_usage",
+        type: "token-usage",
         max_total: maxTotal ?? null,
         max_input: maxInput ?? null,
         max_output: maxOutput ?? null,
@@ -11211,7 +11252,7 @@ function checkLatency(toolName, maxDurationMs, actualDurationMs) {
   };
 }
 var ToolTrajectoryEvaluator = class {
-  kind = "tool_trajectory";
+  kind = "tool-trajectory";
   config;
   constructor(options) {
     this.config = options.config;
@@ -11399,7 +11440,7 @@ var ToolTrajectoryEvaluator = class {
       }
     }
     for (const warning of warnings) {
-      console.warn(`[tool_trajectory] ${warning}`);
+      console.warn(`[tool-trajectory] ${warning}`);
     }
     const effectiveLatencyAssertions = latencyAssertionCount - latencySkips;
     const totalAssertions = expected.length + effectiveLatencyAssertions;
@@ -11475,7 +11516,7 @@ var ToolTrajectoryEvaluator = class {
       misses.push(`Position ${i}: expected ${expected[i].tool}, got nothing`);
     }
     for (const warning of warnings) {
-      console.warn(`[tool_trajectory] ${warning}`);
+      console.warn(`[tool-trajectory] ${warning}`);
     }
     const effectiveLatencyAssertions = latencyAssertionCount - latencySkips;
     const totalAssertions = expected.length + effectiveLatencyAssertions;
@@ -11705,7 +11746,7 @@ function runEqualsAssertion(output, value) {
 
 // src/evaluation/orchestrator.ts
 import { createHash as createHash2, randomUUID as randomUUID7 } from "node:crypto";
-import { mkdir as mkdir12 } from "node:fs/promises";
+import { mkdir as mkdir12, stat as stat7 } from "node:fs/promises";
 import path37 from "node:path";
 import micromatch4 from "micromatch";
 
@@ -11965,7 +12006,7 @@ var llmJudgeFactory = (config, context) => {
   const c = config;
   const { llmJudge, agentTimeoutMs } = context;
   return {
-    kind: "llm_judge",
+    kind: "llm-judge",
     async evaluate(evalContext) {
       const customPrompt = await resolveCustomPrompt(
         c,
@@ -12054,7 +12095,7 @@ var agentJudgeFactory = (config, context) => {
       customPrompt = readFileSync(c.resolvedPromptPath, "utf-8");
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
-      console.warn(`Could not read agent_judge prompt at ${c.resolvedPromptPath}: ${message}`);
+      console.warn(`Could not read agent-judge prompt at ${c.resolvedPromptPath}: ${message}`);
     }
   } else if (c.prompt) {
     customPrompt = c.prompt;
@@ -12064,7 +12105,7 @@ var agentJudgeFactory = (config, context) => {
     judgeTargetProvider = targetResolver(c.target);
     if (!judgeTargetProvider) {
       throw new Error(
-        `agent_judge evaluator '${c.name}': target '${c.target}' not found in targets`
+        `agent-judge evaluator '${c.name}': target '${c.target}' not found in targets`
       );
     }
   }
@@ -12108,7 +12149,7 @@ var regexFactory = (config) => {
   });
 };
 var isJsonFactory = () => {
-  return new DeterministicAssertionEvaluator("is_json", (ctx) => {
+  return new DeterministicAssertionEvaluator("is-json", (ctx) => {
     const result = runIsJsonAssertion(ctx.candidate);
     return {
       score: result.score,
@@ -12136,7 +12177,7 @@ var equalsFactory = (config) => {
 };
 var containsAnyFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("contains_any", (ctx) => {
+  return new DeterministicAssertionEvaluator("contains-any", (ctx) => {
     const result = runContainsAnyAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -12150,7 +12191,7 @@ var containsAnyFactory = (config) => {
 };
 var containsAllFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("contains_all", (ctx) => {
+  return new DeterministicAssertionEvaluator("contains-all", (ctx) => {
     const result = runContainsAllAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -12178,7 +12219,7 @@ var icontainsFactory = (config) => {
 };
 var icontainsAnyFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("icontains_any", (ctx) => {
+  return new DeterministicAssertionEvaluator("icontains-any", (ctx) => {
     const result = runIcontainsAnyAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -12192,7 +12233,7 @@ var icontainsAnyFactory = (config) => {
 };
 var icontainsAllFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("icontains_all", (ctx) => {
+  return new DeterministicAssertionEvaluator("icontains-all", (ctx) => {
     const result = runIcontainsAllAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -12206,7 +12247,7 @@ var icontainsAllFactory = (config) => {
 };
 var startsWithFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("starts_with", (ctx) => {
+  return new DeterministicAssertionEvaluator("starts-with", (ctx) => {
     const result = runStartsWithAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -12220,7 +12261,7 @@ var startsWithFactory = (config) => {
 };
 var endsWithFactory = (config) => {
   const c = config;
-  return new DeterministicAssertionEvaluator("ends_with", (ctx) => {
+  return new DeterministicAssertionEvaluator("ends-with", (ctx) => {
     const result = runEndsWithAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -12234,7 +12275,7 @@ var endsWithFactory = (config) => {
 };
 function createBuiltinRegistry() {
   const registry = new EvaluatorRegistry();
-  registry.register("llm_judge", llmJudgeFactory).register("code", codeFactory).register("composite", compositeFactory).register("tool_trajectory", toolTrajectoryFactory).register("field_accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token_usage", tokenUsageFactory).register("execution_metrics", executionMetricsFactory).register("agent_judge", agentJudgeFactory).register("contains", containsFactory).register("contains_any", containsAnyFactory).register("contains_all", containsAllFactory).register("icontains", icontainsFactory).register("icontains_any", icontainsAnyFactory).register("icontains_all", icontainsAllFactory).register("starts_with", startsWithFactory).register("ends_with", endsWithFactory).register("regex", regexFactory).register("is_json", isJsonFactory).register("equals", equalsFactory);
+  registry.register("llm-judge", llmJudgeFactory).register("code-judge", codeFactory).register("composite", compositeFactory).register("tool-trajectory", toolTrajectoryFactory).register("field-accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token-usage", tokenUsageFactory).register("execution-metrics", executionMetricsFactory).register("agent-judge", agentJudgeFactory).register("contains", containsFactory).register("contains-any", containsAnyFactory).register("contains-all", containsAllFactory).register("icontains", icontainsFactory).register("icontains-any", icontainsAnyFactory).register("icontains-all", icontainsAllFactory).register("starts-with", startsWithFactory).register("ends-with", endsWithFactory).register("regex", regexFactory).register("is-json", isJsonFactory).register("equals", equalsFactory);
   return registry;
 }
 
@@ -12921,7 +12962,8 @@ async function runEvaluation(options) {
     cleanupWorkspaces,
     trials,
     streamCallbacks,
-    totalBudgetUsd
+    totalBudgetUsd,
+    failOnError
   } = options;
   let useCache = options.useCache;
   if (trials && trials.count > 1 && useCache) {
@@ -12979,7 +13021,7 @@ async function runEvaluation(options) {
   };
   if (isAgentProvider(getOrCreateProvider(target)) && !target.judgeTarget) {
     throw new Error(
-      `Target "${target.name}" is an agent provider ("${target.kind}") with no judge_target \u2014 agent providers cannot return structured JSON for judging. Set judge_target to an LLM provider (e.g., azure_base).`
+      `Target "${target.name}" is an agent provider ("${target.kind}") with no judge_target \u2014 agent providers cannot return structured JSON for judging. Set judge_target to an LLM provider (e.g., azure-base).`
     );
   }
   const targetResolver = (name) => {
@@ -13050,7 +13092,7 @@ async function runEvaluation(options) {
   const rawTemplate = suiteWorkspace?.template ?? getWorkspaceTemplate(target);
   const resolvedTemplate = await resolveWorkspaceTemplate(rawTemplate);
   const workspaceTemplate = resolvedTemplate?.dir;
-  const suiteWorkspaceFile = resolvedTemplate?.workspaceFile;
+  let suiteWorkspaceFile = resolvedTemplate?.workspaceFile;
   const isPerTestIsolation = suiteWorkspace?.isolation === "per_test";
   const hasSharedWorkspace = !!(workspaceTemplate || suiteWorkspace?.before_all || suiteWorkspace?.repos?.length && !isPerTestIsolation);
   const requestedWorkers = options.maxConcurrency ?? target.workers ?? 1;
@@ -13071,6 +13113,14 @@ async function runEvaluation(options) {
       const message = error instanceof Error ? error.message : String(error);
       throw new Error(`Failed to create shared workspace: ${message}`);
     }
+    if (suiteWorkspaceFile && sharedWorkspacePath) {
+      const copiedWorkspaceFile = path37.join(sharedWorkspacePath, path37.basename(suiteWorkspaceFile));
+      try {
+        await stat7(copiedWorkspaceFile);
+        suiteWorkspaceFile = copiedWorkspaceFile;
+      } catch {
+      }
+    }
   } else if (suiteWorkspace?.before_all || suiteWorkspace?.repos?.length && !isPerTestIsolation) {
     sharedWorkspacePath = getWorkspacePath(evalRunId, "shared");
     await mkdir12(sharedWorkspacePath, { recursive: true });
@@ -13117,6 +13167,7 @@ async function runEvaluation(options) {
   let beforeAllOutputAttached = false;
   let cumulativeBudgetCost = 0;
   let budgetExhausted = false;
+  let failOnErrorTriggered = false;
   const promises = filteredEvalCases.map(
     (evalCase) => limit(async () => {
       const workerId = nextWorkerId++;
@@ -13155,6 +13206,37 @@ async function runEvaluation(options) {
         }
         return budgetResult;
       }
+      if (failOnError === true && failOnErrorTriggered) {
+        const errorMsg = "Halted: execution error encountered with fail_on_error enabled";
+        const haltResult = {
+          timestamp: (now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
+          testId: evalCase.id,
+          dataset: evalCase.dataset,
+          score: 0,
+          hits: [],
+          misses: [],
+          answer: "",
+          target: target.name,
+          error: errorMsg,
+          executionStatus: "execution_error",
+          failureStage: "setup",
+          failureReasonCode: "error_threshold_exceeded",
+          executionError: { message: errorMsg, stage: "setup" }
+        };
+        if (onProgress) {
+          await onProgress({
+            workerId,
+            testId: evalCase.id,
+            status: "failed",
+            completedAt: Date.now(),
+            error: haltResult.error
+          });
+        }
+        if (onResult) {
+          await onResult(haltResult);
+        }
+        return haltResult;
+      }
       if (onProgress) {
         await onProgress({
           workerId,
@@ -13207,6 +13289,9 @@ async function runEvaluation(options) {
             }
           }
         }
+        if (failOnError === true && result.executionStatus === "execution_error") {
+          failOnErrorTriggered = true;
+        }
         if (beforeAllOutput && !beforeAllOutputAttached) {
           result = { ...result, beforeAllOutput };
           beforeAllOutputAttached = true;
@@ -13514,6 +13599,14 @@ async function runEvalCase(options) {
           "template_error"
         );
       }
+      if (caseWorkspaceFile && workspacePath) {
+        const copiedFile = path37.join(workspacePath, path37.basename(caseWorkspaceFile));
+        try {
+          await stat7(copiedFile);
+          caseWorkspaceFile = copiedFile;
+        } catch {
+        }
+      }
     }
     if (!workspacePath && (evalCase.workspace?.before_all || evalCase.workspace?.repos?.length) && evalRunId) {
       workspacePath = getWorkspacePath(evalRunId, evalCase.id);
@@ -14023,8 +14116,8 @@ async function runEvaluatorsForCase(options) {
       workspacePath
     });
   }
-  const evaluatorKind = evalCase.evaluator ?? "llm_judge";
-  const activeEvaluator = evaluators[evaluatorKind] ?? evaluators.llm_judge;
+  const evaluatorKind = evalCase.evaluator ?? "llm-judge";
+  const activeEvaluator = evaluators[evaluatorKind] ?? evaluators["llm-judge"];
   if (!activeEvaluator) {
     throw new Error(`No evaluator registered for kind '${evaluatorKind}'`);
   }
@@ -14107,25 +14200,24 @@ async function runEvaluatorList(options) {
     availableTargets,
     agentTimeoutMs,
     evalFileDir,
-    llmJudge: evaluatorRegistry.llm_judge,
+    llmJudge: evaluatorRegistry["llm-judge"],
     registry: typeRegistry
   };
   for (const evaluatorConfig of evaluators ?? []) {
     try {
       const evaluatorInstance = await typeRegistry.create(evaluatorConfig, dispatchContext);
       const score2 = await evaluatorInstance.evaluate(evalContext);
-      const resultType = evaluatorConfig.type === "code" ? "code_judge" : evaluatorConfig.type;
       const weight = evaluatorConfig.weight ?? 1;
       scored.push({
         score: score2,
         name: evaluatorConfig.name,
-        type: resultType,
+        type: evaluatorConfig.type,
         weight,
         ...evaluatorConfig.required !== void 0 ? { required: evaluatorConfig.required } : {}
       });
       scores.push({
         name: evaluatorConfig.name,
-        type: resultType,
+        type: evaluatorConfig.type,
         score: score2.score,
         weight,
         verdict: score2.verdict,
@@ -14147,18 +14239,17 @@ async function runEvaluatorList(options) {
         expectedAspectCount: 1,
         reasoning: message
       };
-      const resultType = evaluatorConfig.type === "code" ? "code_judge" : evaluatorConfig.type;
       const weight = evaluatorConfig.weight ?? 1;
       scored.push({
         score: fallbackScore,
         name: evaluatorConfig.name ?? "unknown",
-        type: resultType ?? "llm_judge",
+        type: evaluatorConfig.type ?? "llm-judge",
         weight,
         ...evaluatorConfig.required !== void 0 ? { required: evaluatorConfig.required } : {}
       });
       scores.push({
         name: evaluatorConfig.name ?? "unknown",
-        type: resultType ?? "llm_judge",
+        type: evaluatorConfig.type ?? "llm-judge",
         score: 0,
         weight,
         verdict: "fail",
@@ -14219,7 +14310,7 @@ function filterEvalCases(evalCases, filter) {
   return evalCases.filter((evalCase) => micromatch4.isMatch(evalCase.id, filter));
 }
 function buildEvaluatorRegistry(overrides, resolveJudgeProvider) {
-  const llmJudge = overrides?.llm_judge ?? new LlmJudgeEvaluator({
+  const llmJudge = overrides?.["llm-judge"] ?? new LlmJudgeEvaluator({
     resolveJudgeProvider: async (context) => {
       if (context.judgeProvider) {
         return context.judgeProvider;
@@ -14229,7 +14320,7 @@ function buildEvaluatorRegistry(overrides, resolveJudgeProvider) {
   });
   return {
     ...overrides,
-    llm_judge: llmJudge
+    "llm-judge": llmJudge
   };
 }
 async function invokeProvider(provider, options) {
@@ -14489,12 +14580,7 @@ async function evaluate(config) {
   };
 }
 function mapAssertionType(type) {
-  switch (type) {
-    case "code_judge":
-      return "code";
-    default:
-      return type;
-  }
+  return type.replace(/_/g, "-");
 }
 function computeSummary(results, durationMs) {
   const total = results.length;
@@ -15268,6 +15354,7 @@ export {
   executeWorkspaceScript,
   explorationRatio,
   extractCacheConfig,
+  extractFailOnError,
   extractJsonBlob,
   extractTargetFromSuite,
   extractTargetsFromSuite,