agentv 2.12.0 → 2.14.0-next.1

package/dist/{chunk-LUHCYBMD.js → chunk-OQN2GDEU.js}

@@ -148,7 +148,7 @@ var require_dist = __commonJS({
   }
 });
 
-// ../../packages/core/dist/chunk-7HPKTRFZ.js
+// ../../packages/core/dist/chunk-N55K52OO.js
 import { constants } from "node:fs";
 import { access, readFile } from "node:fs/promises";
 import path from "node:path";
@@ -4195,7 +4195,7 @@ var coerce = {
 };
 var NEVER = INVALID;
 
-// ../../packages/core/dist/chunk-7HPKTRFZ.js
+// ../../packages/core/dist/chunk-N55K52OO.js
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
 var TEST_MESSAGE_ROLES = TEST_MESSAGE_ROLE_VALUES;
 var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
@@ -4243,27 +4243,27 @@ function isTestMessage(value) {
   return false;
 }
 var EVALUATOR_KIND_VALUES = [
-  "code_judge",
-  "llm_judge",
+  "code-judge",
+  "llm-judge",
   "rubric",
   "composite",
-  "tool_trajectory",
-  "field_accuracy",
+  "tool-trajectory",
+  "field-accuracy",
   "latency",
   "cost",
-  "token_usage",
-  "execution_metrics",
-  "agent_judge",
+  "token-usage",
+  "execution-metrics",
+  "agent-judge",
   "contains",
-  "contains_any",
-  "contains_all",
+  "contains-any",
+  "contains-all",
   "icontains",
-  "icontains_any",
-  "icontains_all",
-  "starts_with",
-  "ends_with",
+  "icontains-any",
+  "icontains-all",
+  "starts-with",
+  "ends-with",
   "regex",
-  "is_json",
+  "is-json",
   "equals",
   "rubrics"
 ];
@@ -33960,7 +33960,7 @@ import { createServer } from "node:http";
 import fs2 from "node:fs/promises";
 import path30 from "node:path";
 import { createHash as createHash2, randomUUID as randomUUID7 } from "node:crypto";
-import { mkdir as mkdir12 } from "node:fs/promises";
+import { mkdir as mkdir12, stat as stat7 } from "node:fs/promises";
 import path37 from "node:path";
 import micromatch4 from "micromatch";
 import { readFileSync } from "node:fs";
@@ -34331,6 +34331,11 @@ async function loadConfig(evalFilePath, repoRoot) {
         continue;
       }
       const config2 = parsed;
+      const requiredVersion = parsed.required_version;
+      if (requiredVersion !== void 0 && typeof requiredVersion !== "string") {
+        logWarning(`Invalid required_version in ${configPath}, expected string`);
+        continue;
+      }
       const guidelinePatterns = config2.guideline_patterns;
       if (guidelinePatterns !== void 0 && !Array.isArray(guidelinePatterns)) {
         logWarning(`Invalid guideline_patterns in ${configPath}, expected array`);
@@ -34354,6 +34359,7 @@ async function loadConfig(evalFilePath, repoRoot) {
         configPath
       );
       return {
+        required_version: requiredVersion,
         guideline_patterns: guidelinePatterns,
         eval_patterns: evalPatterns,
         execution: executionDefaults
@@ -34497,6 +34503,22 @@ function extractTotalBudgetUsd(suite) {
   );
   return void 0;
 }
+function extractFailOnError(suite) {
+  const execution = suite.execution;
+  if (!execution || typeof execution !== "object" || Array.isArray(execution)) {
+    return void 0;
+  }
+  const executionObj = execution;
+  const raw = executionObj.fail_on_error ?? executionObj.failOnError;
+  if (raw === void 0 || raw === null) {
+    return void 0;
+  }
+  if (typeof raw === "boolean") {
+    return raw;
+  }
+  logWarning(`Invalid execution.fail_on_error: ${raw}. Must be true or false. Ignoring.`);
+  return void 0;
+}
 function parseExecutionDefaults(raw, configPath) {
   if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
     return void 0;
@@ -34583,6 +34605,9 @@ function validateTemplateVariables(content, source) {
 }
 var ANSI_YELLOW4 = "\x1B[33m";
 var ANSI_RESET4 = "\x1B[0m";
+function normalizeEvaluatorType(type) {
+  return type.replace(/_/g, "-");
+}
 async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId) {
   const execution = rawEvalCase.execution;
   const executionObject = isJsonObject2(execution) ? execution : void 0;
@@ -34613,7 +34638,8 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       continue;
     }
     const rawName = asString(rawEvaluator.name);
-    const typeValue = rawEvaluator.type;
+    const rawType = rawEvaluator.type;
+    const typeValue = typeof rawType === "string" ? normalizeEvaluatorType(rawType) : rawType;
     const isCustomType = typeof typeValue === "string" && !isEvaluatorKind(typeValue);
     if (typeof typeValue !== "string") {
       logWarning2(`Skipping evaluator with invalid type in '${evalId}'`);
@@ -34646,25 +34672,25 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "code_judge") {
+    if (typeValue === "code-judge") {
       let command;
       const rawCommand = rawEvaluator.command ?? rawEvaluator.script;
       if (typeof rawCommand === "string") {
         const trimmed = rawCommand.trim();
         if (trimmed.length === 0) {
           throw new Error(
-            `Invalid code_judge command for evaluator '${name16}' in '${evalId}': command cannot be empty`
+            `Invalid code-judge command for evaluator '${name16}' in '${evalId}': command cannot be empty`
           );
         }
         command = parseCommandToArgv(trimmed);
       } else {
         command = asStringArray(
           rawCommand,
-          `code_judge command for evaluator '${name16}' in '${evalId}'`
+          `code-judge command for evaluator '${name16}' in '${evalId}'`
         );
       }
       if (!command) {
-        logWarning2(`Skipping code_judge evaluator '${name16}' in '${evalId}': missing command`);
+        logWarning2(`Skipping code-judge evaluator '${name16}' in '${evalId}': missing command`);
         continue;
       }
       const weight2 = validateWeight(rawEvaluator.weight, name16, evalId);
@@ -34725,7 +34751,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       }
       evaluators.push({
         name: name16,
-        type: "code",
+        type: "code-judge",
         command,
         cwd,
         resolvedCwd,
@@ -34751,7 +34777,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
         continue;
       }
       const aggregatorType = asString(rawAggregator.type);
-      if (aggregatorType !== "weighted_average" && aggregatorType !== "code_judge" && aggregatorType !== "llm_judge" && aggregatorType !== "threshold") {
+      if (aggregatorType !== "weighted_average" && aggregatorType !== "code-judge" && aggregatorType !== "llm-judge" && aggregatorType !== "threshold") {
         logWarning2(
           `Skipping composite evaluator '${name16}' in '${evalId}': invalid aggregator type '${aggregatorType}'`
         );
@@ -34800,16 +34826,16 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
           type: "weighted_average",
           ...Object.keys(parsedWeights).length > 0 ? { weights: parsedWeights } : {}
         };
-      } else if (aggregatorType === "code_judge") {
+      } else if (aggregatorType === "code-judge") {
         const aggregatorPath = asString(rawAggregator.path);
         if (!aggregatorPath) {
           logWarning2(
-            `Skipping composite evaluator '${name16}' in '${evalId}': code_judge aggregator missing path`
+            `Skipping composite evaluator '${name16}' in '${evalId}': code-judge aggregator missing path`
           );
           continue;
         }
         aggregator = {
-          type: "code_judge",
+          type: "code-judge",
           path: aggregatorPath,
           cwd: searchRoots[0]
         };
@@ -34835,7 +34861,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
           }
         }
         aggregator = {
-          type: "llm_judge",
+          type: "llm-judge",
           ...aggregatorPrompt ? { prompt: aggregatorPrompt } : {},
           ...promptPath2 ? { promptPath: promptPath2 } : {}
         };
@@ -34853,11 +34879,11 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "tool_trajectory") {
+    if (typeValue === "tool-trajectory") {
       const mode = asString(rawEvaluator.mode);
       if (mode !== "any_order" && mode !== "in_order" && mode !== "exact" && mode !== "subset" && mode !== "superset") {
         logWarning2(
-          `Skipping tool_trajectory evaluator '${name16}' in '${evalId}': invalid mode '${mode}' (must be any_order, in_order, exact, subset, or superset)`
+          `Skipping tool-trajectory evaluator '${name16}' in '${evalId}': invalid mode '${mode}' (must be any_order, in_order, exact, subset, or superset)`
         );
         continue;
       }
@@ -34866,7 +34892,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       if (rawMinimums !== void 0) {
         if (!isJsonObject2(rawMinimums)) {
           logWarning2(
-            `Skipping tool_trajectory evaluator '${name16}' in '${evalId}': minimums must be an object`
+            `Skipping tool-trajectory evaluator '${name16}' in '${evalId}': minimums must be an object`
           );
           continue;
         }
@@ -34892,7 +34918,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
             argsMatch2 = rawArgsMatch;
           } else {
             logWarning2(
-              `Invalid args_match '${rawArgsMatch}' for tool_trajectory evaluator '${name16}' in '${evalId}': must be exact, superset, subset, ignore, or a string array`
+              `Invalid args_match '${rawArgsMatch}' for tool-trajectory evaluator '${name16}' in '${evalId}': must be exact, superset, subset, ignore, or a string array`
             );
           }
         }
@@ -34902,7 +34928,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       if (rawExpected !== void 0) {
         if (!Array.isArray(rawExpected)) {
           logWarning2(
-            `Skipping tool_trajectory evaluator '${name16}' in '${evalId}': expected must be an array`
+            `Skipping tool-trajectory evaluator '${name16}' in '${evalId}': expected must be an array`
           );
           continue;
         }
@@ -34948,13 +34974,13 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       }
       if (mode === "any_order" && !minimums) {
         logWarning2(
-          `Skipping tool_trajectory evaluator '${name16}' in '${evalId}': any_order mode requires minimums`
+          `Skipping tool-trajectory evaluator '${name16}' in '${evalId}': any_order mode requires minimums`
         );
         continue;
       }
       if ((mode === "in_order" || mode === "exact" || mode === "subset" || mode === "superset") && !expected) {
         logWarning2(
-          `Skipping tool_trajectory evaluator '${name16}' in '${evalId}': ${mode} mode requires expected`
+          `Skipping tool-trajectory evaluator '${name16}' in '${evalId}': ${mode} mode requires expected`
         );
         continue;
       }
@@ -34962,7 +34988,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required22 = parseRequired(rawEvaluator.required);
       const config22 = {
         name: name16,
-        type: "tool_trajectory",
+        type: "tool-trajectory",
         mode,
         ...minimums ? { minimums } : {},
         ...expected ? { expected } : {},
@@ -34974,17 +35000,17 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       evaluators.push(config22);
       continue;
     }
-    if (typeValue === "field_accuracy") {
+    if (typeValue === "field-accuracy") {
       const rawFields = rawEvaluator.fields;
       if (!Array.isArray(rawFields)) {
         logWarning2(
-          `Skipping field_accuracy evaluator '${name16}' in '${evalId}': missing fields array`
+          `Skipping field-accuracy evaluator '${name16}' in '${evalId}': missing fields array`
         );
         continue;
       }
       if (rawFields.length === 0) {
         logWarning2(
-          `Skipping field_accuracy evaluator '${name16}' in '${evalId}': fields array is empty`
+          `Skipping field-accuracy evaluator '${name16}' in '${evalId}': fields array is empty`
         );
         continue;
       }
@@ -34992,7 +35018,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       for (const rawField of rawFields) {
         if (!isJsonObject2(rawField)) {
           logWarning2(
-            `Skipping invalid field entry in field_accuracy evaluator '${name16}' (expected object)`
+            `Skipping invalid field entry in field-accuracy evaluator '${name16}' (expected object)`
           );
           continue;
         }
@@ -35000,13 +35026,13 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
         const match = asString(rawField.match);
         if (!fieldPath) {
           logWarning2(
-            `Skipping field without path in field_accuracy evaluator '${name16}' in '${evalId}'`
+            `Skipping field without path in field-accuracy evaluator '${name16}' in '${evalId}'`
           );
           continue;
         }
         if (!match || !isValidFieldMatchType(match)) {
           logWarning2(
-            `Skipping field '${fieldPath}' with invalid match type '${match}' in evaluator '${name16}' (must be exact, numeric_tolerance, or date). For fuzzy matching, use a code_judge evaluator.`
+            `Skipping field '${fieldPath}' with invalid match type '${match}' in evaluator '${name16}' (must be exact, numeric_tolerance, or date). For fuzzy matching, use a code-judge evaluator.`
           );
           continue;
         }
@@ -35023,7 +35049,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       }
       if (fields.length === 0) {
         logWarning2(
-          `Skipping field_accuracy evaluator '${name16}' in '${evalId}': no valid fields found`
+          `Skipping field-accuracy evaluator '${name16}' in '${evalId}': no valid fields found`
         );
         continue;
       }
@@ -35033,7 +35059,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required22 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name: name16,
-        type: "field_accuracy",
+        type: "field-accuracy",
         fields,
         ...validAggregation ? { aggregation: validAggregation } : {},
         ...weight2 !== void 0 ? { weight: weight2 } : {},
@@ -35082,7 +35108,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "token_usage") {
+    if (typeValue === "token-usage") {
       const maxTotal = rawEvaluator.max_total ?? rawEvaluator.maxTotal;
       const maxInput = rawEvaluator.max_input ?? rawEvaluator.maxInput;
       const maxOutput = rawEvaluator.max_output ?? rawEvaluator.maxOutput;
@@ -35096,7 +35122,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
         if (raw === void 0) continue;
         if (typeof raw !== "number" || !Number.isFinite(raw) || raw < 0) {
           logWarning2(
-            `Skipping token_usage evaluator '${name16}' in '${evalId}': ${key} must be a non-negative finite number`
+            `Skipping token-usage evaluator '${name16}' in '${evalId}': ${key} must be a non-negative finite number`
           );
           continue;
         }
@@ -35104,7 +35130,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       }
       if (validLimits.max_total === void 0 && validLimits.max_input === void 0 && validLimits.max_output === void 0) {
         logWarning2(
-          `Skipping token_usage evaluator '${name16}' in '${evalId}': must set at least one of max_total, max_input, max_output`
+          `Skipping token-usage evaluator '${name16}' in '${evalId}': must set at least one of max_total, max_input, max_output`
         );
         continue;
       }
@@ -35112,7 +35138,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required22 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name: name16,
-        type: "token_usage",
+        type: "token-usage",
         ...validLimits,
         ...weight2 !== void 0 ? { weight: weight2 } : {},
         ...required22 !== void 0 ? { required: required22 } : {},
@@ -35120,7 +35146,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "execution_metrics") {
+    if (typeValue === "execution-metrics") {
       const maxToolCalls = rawEvaluator.max_tool_calls ?? rawEvaluator.maxToolCalls;
       const maxLlmCalls = rawEvaluator.max_llm_calls ?? rawEvaluator.maxLlmCalls;
       const maxTokens = rawEvaluator.max_tokens ?? rawEvaluator.maxTokens;
@@ -35143,7 +35169,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
         if (raw === void 0) continue;
         if (typeof raw !== "number" || !Number.isFinite(raw) || raw < 0) {
           logWarning2(
-            `Skipping execution_metrics evaluator '${name16}' in '${evalId}': ${key} must be a non-negative finite number`
+            `Skipping execution-metrics evaluator '${name16}' in '${evalId}': ${key} must be a non-negative finite number`
           );
           hasError = true;
           break;
@@ -35156,7 +35182,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const hasThreshold = validThresholds.max_tool_calls !== void 0 || validThresholds.max_llm_calls !== void 0 || validThresholds.max_tokens !== void 0 || validThresholds.max_cost_usd !== void 0 || validThresholds.max_duration_ms !== void 0 || validThresholds.target_exploration_ratio !== void 0;
       if (!hasThreshold) {
         logWarning2(
-          `Skipping execution_metrics evaluator '${name16}' in '${evalId}': must set at least one threshold (max_tool_calls, max_llm_calls, max_tokens, max_cost_usd, max_duration_ms, or target_exploration_ratio)`
+          `Skipping execution-metrics evaluator '${name16}' in '${evalId}': must set at least one threshold (max_tool_calls, max_llm_calls, max_tokens, max_cost_usd, max_duration_ms, or target_exploration_ratio)`
         );
         continue;
       }
@@ -35164,7 +35190,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required22 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name: name16,
-        type: "execution_metrics",
+        type: "execution-metrics",
         ...validThresholds,
         ...weight2 !== void 0 ? { weight: weight2 } : {},
         ...required22 !== void 0 ? { required: required22 } : {},
@@ -35172,13 +35198,13 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "agent_judge") {
+    if (typeValue === "agent-judge") {
       const rawMaxSteps = rawEvaluator.max_steps ?? rawEvaluator.maxSteps;
       let maxSteps;
       if (rawMaxSteps !== void 0) {
         if (typeof rawMaxSteps !== "number" || !Number.isInteger(rawMaxSteps) || rawMaxSteps < 1 || rawMaxSteps > 50) {
           logWarning2(
-            `Skipping agent_judge evaluator '${name16}' in '${evalId}': max_steps must be an integer 1-50`
+            `Skipping agent-judge evaluator '${name16}' in '${evalId}': max_steps must be an integer 1-50`
           );
           continue;
         }
@@ -35189,7 +35215,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       if (rawTemperature !== void 0) {
         if (typeof rawTemperature !== "number" || rawTemperature < 0 || rawTemperature > 2) {
           logWarning2(
-            `Skipping agent_judge evaluator '${name16}' in '${evalId}': temperature must be a number 0-2`
+            `Skipping agent-judge evaluator '${name16}' in '${evalId}': temperature must be a number 0-2`
           );
           continue;
         }
@@ -35212,7 +35238,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required22 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name: name16,
-        type: "agent_judge",
+        type: "agent-judge",
         ...agentPrompt ? { prompt: agentPrompt } : {},
         ...agentPromptPath ? { promptPath: agentPromptPath, resolvedPromptPath: agentPromptPath } : {},
         ...agentParsedRubrics && agentParsedRubrics.length > 0 ? { rubrics: agentParsedRubrics } : {},
@@ -35243,7 +35269,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "contains_any" || typeValue === "contains_all") {
+    if (typeValue === "contains-any" || typeValue === "contains-all") {
       const value = asStringArrayStrict(rawEvaluator.value);
       if (!value || value.length === 0) {
         logWarning2(
@@ -35281,7 +35307,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "icontains_any" || typeValue === "icontains_all") {
+    if (typeValue === "icontains-any" || typeValue === "icontains-all") {
       const value = asStringArrayStrict(rawEvaluator.value);
       if (!value || value.length === 0) {
         logWarning2(
@@ -35301,7 +35327,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "starts_with" || typeValue === "ends_with") {
+    if (typeValue === "starts-with" || typeValue === "ends-with") {
       const value = asString(rawEvaluator.value);
       if (!value) {
         logWarning2(`Skipping ${typeValue} evaluator '${name16}' in '${evalId}': missing value`);
@@ -35339,12 +35365,12 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       });
       continue;
     }
-    if (typeValue === "is_json") {
+    if (typeValue === "is-json") {
       const weight2 = validateWeight(rawEvaluator.weight, name16, evalId);
       const required22 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name: name16,
-        type: "is_json",
+        type: "is-json",
         ...weight2 !== void 0 ? { weight: weight2 } : {},
         ...required22 !== void 0 ? { required: required22 } : {},
         ...negate !== void 0 ? { negate } : {}
@@ -35392,7 +35418,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required22 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name: name16,
-        type: "llm_judge",
+        type: "llm-judge",
         rubrics: parsedCriteria,
         ...weight2 !== void 0 ? { weight: weight2 } : {},
         ...required22 !== void 0 ? { required: required22 } : {},
@@ -35459,7 +35485,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const required22 = parseRequired(rawEvaluator.required);
       evaluators.push({
         name: name16,
-        type: "llm_judge",
+        type: "llm-judge",
         rubrics: parsedRubrics,
         ...weight2 !== void 0 ? { weight: weight2 } : {},
         ...required22 !== void 0 ? { required: required22 } : {},
@@ -35491,7 +35517,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
     const finalConfig = promptScriptConfig ?? (Object.keys(mergedConfig).length > 0 ? mergedConfig : void 0);
     evaluators.push({
       name: name16,
-      type: "llm_judge",
+      type: "llm-judge",
       prompt,
       promptPath,
       ...promptPath ? { resolvedPromptPath: promptPath } : {},
@@ -35507,15 +35533,15 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
 }
 var ASSERTION_TYPES = /* @__PURE__ */ new Set([
   "contains",
-  "contains_any",
-  "contains_all",
+  "contains-any",
+  "contains-all",
   "icontains",
-  "icontains_any",
-  "icontains_all",
-  "starts_with",
-  "ends_with",
+  "icontains-any",
+  "icontains-all",
+  "starts-with",
+  "ends-with",
   "regex",
-  "is_json",
+  "is-json",
   "equals",
   "rubrics"
 ]);
@@ -35528,24 +35554,24 @@ function generateAssertionName(typeValue, rawEvaluator) {
   switch (typeValue) {
     case "contains":
       return value ? `contains-${value}` : "contains";
-    case "contains_any":
-      return arrayValue ? `contains_any-${arrayValue.length}` : "contains_any";
-    case "contains_all":
-      return arrayValue ? `contains_all-${arrayValue.length}` : "contains_all";
+    case "contains-any":
+      return arrayValue ? `contains-any-${arrayValue.length}` : "contains-any";
+    case "contains-all":
+      return arrayValue ? `contains-all-${arrayValue.length}` : "contains-all";
     case "icontains":
       return value ? `icontains-${value}` : "icontains";
-    case "icontains_any":
-      return arrayValue ? `icontains_any-${arrayValue.length}` : "icontains_any";
-    case "icontains_all":
-      return arrayValue ? `icontains_all-${arrayValue.length}` : "icontains_all";
-    case "starts_with":
-      return value ? `starts_with-${value}` : "starts_with";
-    case "ends_with":
-      return value ? `ends_with-${value}` : "ends_with";
+    case "icontains-any":
+      return arrayValue ? `icontains-any-${arrayValue.length}` : "icontains-any";
+    case "icontains-all":
+      return arrayValue ? `icontains-all-${arrayValue.length}` : "icontains-all";
+    case "starts-with":
+      return value ? `starts-with-${value}` : "starts-with";
+    case "ends-with":
+      return value ? `ends-with-${value}` : "ends-with";
     case "regex":
       return value ? `regex-${value.length > 30 ? value.slice(0, 30) : value}` : "regex";
-    case "is_json":
-      return "is_json";
+    case "is-json":
+      return "is-json";
     case "equals":
       return value ? `equals-${value}` : "equals";
     case "rubrics":
@@ -35558,8 +35584,9 @@ function coerceEvaluator(candidate, contextId) {
   if (typeof candidate !== "string") {
     return void 0;
   }
-  if (isEvaluatorKind(candidate)) {
-    return candidate;
+  const normalized = normalizeEvaluatorType(candidate);
+  if (isEvaluatorKind(normalized)) {
+    return normalized;
   }
   logWarning2(`Unknown evaluator '${candidate}' in ${contextId}, falling back to default`);
   return void 0;
@@ -35605,6 +35632,16 @@ function parseCommandToArgv(command) {
 function isJsonObject2(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
+var CRITERIA_CONSUMER_TYPES = /* @__PURE__ */ new Set(["llm-judge", "agent-judge", "code-judge"]);
+function warnUnconsumedCriteria(criteria, evaluators, testId) {
+  if (!criteria?.trim() || !evaluators || evaluators.length === 0) return;
+  const hasConsumer = evaluators.some((e) => CRITERIA_CONSUMER_TYPES.has(e.type));
+  if (!hasConsumer) {
+    logWarning2(
+      `Test '${testId}': criteria is defined but no evaluator in assert will evaluate it. Add 'type: llm-judge' to assert, or remove criteria if it is documentation-only.`
+    );
+  }
+}
 function logWarning2(message, details) {
   if (details && details.length > 0) {
     const detailBlock = details.join("\n");
@@ -35854,7 +35891,7 @@ function parseInlineRubrics(rawRubrics) {
   }
   return {
     name: "rubric",
-    type: "llm_judge",
+    type: "llm-judge",
     rubrics: rubricItems
   };
 }
@@ -36221,7 +36258,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
   const rawCases = parseJsonlContent(rawFile, evalFilePath);
   const fallbackDataset = path6.basename(absoluteTestPath, ".jsonl") || "eval";
   const datasetName = sidecar.dataset && sidecar.dataset.trim().length > 0 ? sidecar.dataset : fallbackDataset;
-  const globalEvaluator = coerceEvaluator(sidecar.evaluator, "sidecar") ?? "llm_judge";
+  const globalEvaluator = coerceEvaluator(sidecar.evaluator, "sidecar") ?? "llm-judge";
   const globalExecution = sidecar.execution;
   if (verbose) {
     console.log(`
@@ -36309,6 +36346,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
         evaluators = evaluators ? [rubricEvaluator, ...evaluators] : [rubricEvaluator];
       }
     }
+    warnUnconsumedCriteria(outcome, evaluators, id ?? "unknown");
     const userFilePaths = [];
     for (const segment of inputSegments) {
       if (segment.type === "file" && typeof segment.resolvedPath === "string") {
@@ -36653,13 +36691,15 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
   }
   const { tests, parsed } = await loadTestsFromYaml(evalFilePath, repoRoot, options);
   const metadata = parseMetadata(parsed);
+  const failOnError = extractFailOnError(parsed);
   return {
     tests,
     trials: extractTrialsConfig(parsed),
     targets: extractTargetsFromSuite(parsed),
     cacheConfig: extractCacheConfig(parsed),
     totalBudgetUsd: extractTotalBudgetUsd(parsed),
-    ...metadata !== void 0 && { metadata }
+    ...metadata !== void 0 && { metadata },
+    ...failOnError !== void 0 && { failOnError }
   };
 }
 var loadEvalSuite = loadTestSuite;
@@ -36690,7 +36730,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
   const fallbackDataset = path8.basename(absoluteTestPath).replace(/\.ya?ml$/i, "") || "eval";
   const datasetName = datasetNameFromSuite && datasetNameFromSuite.length > 0 ? datasetNameFromSuite : fallbackDataset;
   const rawTestcases = resolveTests(suite);
-  const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm_judge";
+  const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm-judge";
   const evalFileDir = path8.dirname(absoluteTestPath);
   let expandedTestcases;
   if (typeof rawTestcases === "string") {
@@ -36787,6 +36827,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
         evaluators = evaluators ? [rubricEvaluator, ...evaluators] : [rubricEvaluator];
       }
     }
+    warnUnconsumedCriteria(outcome, evaluators, id ?? "unknown");
     const userFilePaths = [];
     for (const segment of inputSegments) {
       if (segment.type === "file" && typeof segment.resolvedPath === "string") {
@@ -42588,7 +42629,7 @@ function toCamelCaseDeep(obj) {
 }
 var FILE_BACKED_OUTPUT_THRESHOLD = 5e4;
 var CodeEvaluator = class {
-  kind = "code";
+  kind = "code-judge";
   command;
   cwd;
   agentTimeoutMs;
@@ -42789,7 +42830,7 @@ var scoreRangeEvaluationSchema = external_exports.object({
   overall_reasoning: external_exports.string().describe("Overall assessment summary (1-2 sentences)").optional()
 });
 var LlmJudgeEvaluator = class {
-  kind = "llm_judge";
+  kind = "llm-judge";
   resolveJudgeProvider;
   maxOutputTokens;
   temperature;
@@ -42806,7 +42847,7 @@ var LlmJudgeEvaluator = class {
       throw new Error("No judge provider available for LLM grading");
     }
     const config2 = context.evaluator;
-    if (config2?.type === "llm_judge" && config2.rubrics && config2.rubrics.length > 0) {
+    if (config2?.type === "llm-judge" && config2.rubrics && config2.rubrics.length > 0) {
       return this.evaluateWithRubrics(context, judgeProvider, config2.rubrics);
     }
     return this.evaluateFreeform(context, judgeProvider);
@@ -42880,7 +42921,7 @@ ${context.fileChanges}`;
   async evaluateWithRubrics(context, judgeProvider, rubrics) {
     if (!rubrics || rubrics.length === 0) {
       throw new Error(
-        `No rubrics found for evaluator "${context.evaluator?.name ?? "llm_judge"}". Run "agentv generate rubrics" first.`
+        `No rubrics found for evaluator "${context.evaluator?.name ?? "llm-judge"}". Run "agentv generate rubrics" first.`
       );
     }
     const hasScoreRanges = rubrics.some((r) => r.score_ranges && r.score_ranges.length > 0);
@@ -43214,9 +43255,9 @@ var CompositeEvaluator = class {
   async aggregate(results, context) {
     const aggregator = this.config.aggregator;
     switch (aggregator.type) {
-      case "code_judge":
+      case "code-judge":
         return this.runCodeAggregator(results, aggregator.path, aggregator.cwd ?? this.cwd);
-      case "llm_judge":
+      case "llm-judge":
         return this.runLlmAggregator(results, context, aggregator);
       case "threshold":
         return this.runThreshold(results, aggregator.threshold);
@@ -43359,7 +43400,7 @@ var CompositeEvaluator = class {
         expectedAspectCount: hits.length + misses.length || 1,
         reasoning,
         evaluatorRawRequest: {
-          aggregator: "code_judge",
+          aggregator: "code-judge",
           script: scriptPath
         },
         scores
@@ -43374,7 +43415,7 @@ var CompositeEvaluator = class {
         expectedAspectCount: 1,
         reasoning: message,
         evaluatorRawRequest: {
-          aggregator: "code_judge",
+          aggregator: "code-judge",
           script: scriptPath,
           error: message
         },
@@ -43405,7 +43446,7 @@ var CompositeEvaluator = class {
     const userPrompt = promptTemplate.replace(/\{\{EVALUATOR_RESULTS_JSON\}\}/g, resultsJson);
     const systemPrompt = buildOutputSchema();
     const evaluatorRawRequest = {
-      aggregator: "llm_judge",
+      aggregator: "llm-judge",
       userPrompt,
       systemPrompt,
       target: judgeProvider.targetName
@@ -43513,7 +43554,7 @@ var CostEvaluator = class {
   }
 };
 var ExecutionMetricsEvaluator = class {
-  kind = "execution_metrics";
+  kind = "execution-metrics";
   config;
   constructor(options) {
     this.config = options.config;
@@ -43539,7 +43580,7 @@ var ExecutionMetricsEvaluator = class {
         expectedAspectCount: 1,
         reasoning: "Execution metrics not available - no trace summary provided",
         evaluatorRawRequest: {
-          type: "execution_metrics",
+          type: "execution-metrics",
           config: this.extractConfiguredThresholds(),
           actual: null
         }
@@ -43648,7 +43689,7 @@ var ExecutionMetricsEvaluator = class {
     if (actualMetrics.exploration_ratio !== void 0) {
       reasoningParts.push(`exploration_ratio=${actualMetrics.exploration_ratio.toFixed(2)}`);
     }
-    const reasoning = reasoningParts.length > 0 ? `execution_metrics ${reasoningParts.join(", ")}` : "No metrics evaluated";
+    const reasoning = reasoningParts.length > 0 ? `execution-metrics ${reasoningParts.join(", ")}` : "No metrics evaluated";
     return {
       score,
       verdict: scoreToVerdict(score),
@@ -43657,7 +43698,7 @@ var ExecutionMetricsEvaluator = class {
       expectedAspectCount: totalChecks || 1,
       reasoning,
       evaluatorRawRequest: {
-        type: "execution_metrics",
+        type: "execution-metrics",
         config: this.extractConfiguredThresholds(),
         actual: this.filterDefinedMetrics(actualMetrics)
       }
@@ -43743,7 +43784,7 @@ var MONTH_NAMES = {
   december: 11
 };
 var FieldAccuracyEvaluator = class {
-  kind = "field_accuracy";
+  kind = "field-accuracy";
   config;
   constructor(options) {
     this.config = options.config;
@@ -44189,7 +44230,7 @@ var BINARY_EXTENSIONS = /* @__PURE__ */ new Set([
   ".dylib"
 ]);
 var AgentJudgeEvaluator = class {
-  kind = "agent_judge";
+  kind = "agent-judge";
   resolveJudgeProvider;
   maxSteps;
   temperature;
@@ -44214,24 +44255,24 @@ var AgentJudgeEvaluator = class {
   async evaluateBuiltIn(context) {
     const judgeProvider = await this.resolveJudgeProvider(context);
     if (!judgeProvider) {
-      throw new Error("No judge provider available for agent_judge evaluation");
+      throw new Error("No judge provider available for agent-judge evaluation");
     }
     const model = judgeProvider.asLanguageModel?.();
     if (!model) {
       throw new Error(
-        `Judge provider '${judgeProvider.targetName}' does not support asLanguageModel() \u2014 required for built-in agent_judge mode`
+        `Judge provider '${judgeProvider.targetName}' does not support asLanguageModel() \u2014 required for built-in agent-judge mode`
       );
     }
     const workspacePath = context.workspacePath;
     if (!workspacePath) {
       throw new Error(
-        "agent_judge evaluator requires a workspace_template target (workspacePath is not set)"
+        "agent-judge evaluator requires a workspace_template target (workspacePath is not set)"
       );
     }
     const systemPrompt = this.buildSystemPrompt(context);
     const userPrompt = this.buildUserPrompt(context);
     const config2 = context.evaluator;
-    const rubrics = config2?.type === "agent_judge" ? config2.rubrics : void 0;
+    const rubrics = config2?.type === "agent-judge" ? config2.rubrics : void 0;
     const fsTools = createFilesystemTools(workspacePath);
     const evaluatorRawRequest = {
       mode: "built-in",
@@ -44262,7 +44303,7 @@ var AgentJudgeEvaluator = class {
         score: 0,
         verdict: "fail",
         hits: [],
-        misses: [`agent_judge built-in evaluation failed: ${message}`],
+        misses: [`agent-judge built-in evaluation failed: ${message}`],
         expectedAspectCount: 1,
         evaluatorRawRequest,
         details: { mode: "built-in", error: message }
@@ -44294,14 +44335,14 @@ var AgentJudgeEvaluator = class {
           score: 0,
           verdict: "fail",
           hits: [],
-          misses: ["agent_judge judge_target returned no assistant response"],
+          misses: ["agent-judge judge_target returned no assistant response"],
           expectedAspectCount: 1,
           evaluatorRawRequest,
           details: { mode: "judge_target", judge_target: provider.targetName }
         };
       }
       const config2 = context.evaluator;
-      const rubrics = config2?.type === "agent_judge" ? config2.rubrics : void 0;
+      const rubrics = config2?.type === "agent-judge" ? config2.rubrics : void 0;
       const details = {
         mode: "judge_target",
         judge_target: provider.targetName
@@ -44313,7 +44354,7 @@ var AgentJudgeEvaluator = class {
         score: 0,
         verdict: "fail",
         hits: [],
-        misses: [`agent_judge judge_target evaluation failed: ${message}`],
+        misses: [`agent-judge judge_target evaluation failed: ${message}`],
         expectedAspectCount: 1,
         evaluatorRawRequest,
         details: {
@@ -44364,7 +44405,7 @@ var AgentJudgeEvaluator = class {
         score: 0,
         verdict: "fail",
         hits: [],
-        misses: ["Failed to parse agent_judge response as valid evaluation JSON"],
+        misses: ["Failed to parse agent-judge response as valid evaluation JSON"],
         expectedAspectCount: 1,
         evaluatorRawRequest,
         details
@@ -44377,7 +44418,7 @@ var AgentJudgeEvaluator = class {
    */
   buildSystemPrompt(context) {
     const config2 = context.evaluator;
-    const rubrics = config2?.type === "agent_judge" ? config2.rubrics : void 0;
+    const rubrics = config2?.type === "agent-judge" ? config2.rubrics : void 0;
     const parts = [
       "You are an expert evaluator with access to the workspace filesystem.",
       "Use the provided tools to investigate the workspace and verify the criteria are met.",
@@ -44408,7 +44449,7 @@ var AgentJudgeEvaluator = class {
       return substituteVariables(this.evaluatorTemplate, variables);
     }
     const config2 = context.evaluator;
-    const rubrics = config2?.type === "agent_judge" ? config2.rubrics : void 0;
+    const rubrics = config2?.type === "agent-judge" ? config2.rubrics : void 0;
     const parts = [
       "Evaluate the candidate answer by investigating the workspace.",
       "",
@@ -44451,7 +44492,7 @@ var AgentJudgeEvaluator = class {
   buildDelegatedPrompt(context) {
     const formattedQuestion = context.promptInputs.question && context.promptInputs.question.trim().length > 0 ? context.promptInputs.question : context.evalCase.question;
     const config2 = context.evaluator;
-    const rubrics = config2?.type === "agent_judge" ? config2.rubrics : void 0;
+    const rubrics = config2?.type === "agent-judge" ? config2.rubrics : void 0;
     if (this.evaluatorTemplate) {
       const variables = {
         [TEMPLATE_VARIABLES.ANSWER]: context.candidate.trim(),
@@ -44533,11 +44574,11 @@ function createFilesystemTools(workspacePath) {
       execute: async (input) => {
         try {
           const resolved = resolveSandboxed(workspacePath, input.path);
-          const stat7 = await fs2.stat(resolved);
-          if (stat7.isDirectory()) {
+          const stat8 = await fs2.stat(resolved);
+          if (stat8.isDirectory()) {
             return { error: `'${input.path}' is a directory, not a file` };
           }
-          const buffer = Buffer.alloc(Math.min(stat7.size, MAX_FILE_SIZE));
+          const buffer = Buffer.alloc(Math.min(stat8.size, MAX_FILE_SIZE));
           const fd = await fs2.open(resolved, "r");
           try {
             await fd.read(buffer, 0, buffer.length, 0);
@@ -44545,8 +44586,8 @@ function createFilesystemTools(workspacePath) {
             await fd.close();
           }
           const content = buffer.toString("utf-8");
-          const truncated = stat7.size > MAX_FILE_SIZE;
-          return { content, truncated, size: stat7.size };
+          const truncated = stat8.size > MAX_FILE_SIZE;
+          return { content, truncated, size: stat8.size };
         } catch (error40) {
           return { error: error40 instanceof Error ? error40.message : String(error40) };
         }
@@ -44590,8 +44631,8 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
       const ext = path30.extname(entry.name).toLowerCase();
       if (BINARY_EXTENSIONS.has(ext)) continue;
       try {
-        const stat7 = await fs2.stat(fullPath);
-        if (stat7.size > MAX_FILE_SIZE) continue;
+        const stat8 = await fs2.stat(fullPath);
+        if (stat8.size > MAX_FILE_SIZE) continue;
         const content = await fs2.readFile(fullPath, "utf-8");
         const lines = content.split("\n");
         for (let i = 0; i < lines.length; i++) {
@@ -44749,7 +44790,7 @@ function assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChan
   };
 }
 var TokenUsageEvaluator = class {
-  kind = "token_usage";
+  kind = "token-usage";
   config;
   constructor(options) {
     this.config = options.config;
@@ -44772,7 +44813,7 @@ var TokenUsageEvaluator = class {
         expectedAspectCount,
         reasoning: "Token usage not reported by provider",
         evaluatorRawRequest: {
-          type: "token_usage",
+          type: "token-usage",
           max_total: maxTotal ?? null,
           max_input: maxInput ?? null,
           max_output: maxOutput ?? null,
@@ -44814,9 +44855,9 @@ var TokenUsageEvaluator = class {
       hits,
       misses,
       expectedAspectCount,
-      reasoning: `token_usage input=${input}, output=${output}, cached=${cached2}, total=${total}`,
+      reasoning: `token-usage input=${input}, output=${output}, cached=${cached2}, total=${total}`,
       evaluatorRawRequest: {
-        type: "token_usage",
+        type: "token-usage",
         max_total: maxTotal ?? null,
         max_input: maxInput ?? null,
         max_output: maxOutput ?? null,
@@ -44899,7 +44940,7 @@ function checkLatency(toolName, maxDurationMs, actualDurationMs) {
   };
 }
 var ToolTrajectoryEvaluator = class {
-  kind = "tool_trajectory";
+  kind = "tool-trajectory";
   config;
   constructor(options) {
     this.config = options.config;
@@ -45087,7 +45128,7 @@ var ToolTrajectoryEvaluator = class {
       }
     }
     for (const warning of warnings) {
-      console.warn(`[tool_trajectory] ${warning}`);
+      console.warn(`[tool-trajectory] ${warning}`);
     }
     const effectiveLatencyAssertions = latencyAssertionCount - latencySkips;
     const totalAssertions = expected.length + effectiveLatencyAssertions;
@@ -45163,7 +45204,7 @@ var ToolTrajectoryEvaluator = class {
       misses.push(`Position ${i}: expected ${expected[i].tool}, got nothing`);
     }
     for (const warning of warnings) {
-      console.warn(`[tool_trajectory] ${warning}`);
+      console.warn(`[tool-trajectory] ${warning}`);
     }
     const effectiveLatencyAssertions = latencyAssertionCount - latencySkips;
     const totalAssertions = expected.length + effectiveLatencyAssertions;
@@ -45631,7 +45672,7 @@ var llmJudgeFactory = (config2, context) => {
   const c = config2;
   const { llmJudge, agentTimeoutMs } = context;
   return {
-    kind: "llm_judge",
+    kind: "llm-judge",
     async evaluate(evalContext) {
       const customPrompt = await resolveCustomPrompt(
         c,
@@ -45720,7 +45761,7 @@ var agentJudgeFactory = (config2, context) => {
       customPrompt = readFileSync(c.resolvedPromptPath, "utf-8");
     } catch (error40) {
       const message = error40 instanceof Error ? error40.message : String(error40);
-      console.warn(`Could not read agent_judge prompt at ${c.resolvedPromptPath}: ${message}`);
+      console.warn(`Could not read agent-judge prompt at ${c.resolvedPromptPath}: ${message}`);
     }
   } else if (c.prompt) {
     customPrompt = c.prompt;
@@ -45730,7 +45771,7 @@ var agentJudgeFactory = (config2, context) => {
     judgeTargetProvider = targetResolver(c.target);
     if (!judgeTargetProvider) {
       throw new Error(
-        `agent_judge evaluator '${c.name}': target '${c.target}' not found in targets`
+        `agent-judge evaluator '${c.name}': target '${c.target}' not found in targets`
       );
     }
   }
@@ -45774,7 +45815,7 @@ var regexFactory = (config2) => {
   });
 };
 var isJsonFactory = () => {
-  return new DeterministicAssertionEvaluator("is_json", (ctx) => {
+  return new DeterministicAssertionEvaluator("is-json", (ctx) => {
     const result = runIsJsonAssertion(ctx.candidate);
     return {
       score: result.score,
@@ -45802,7 +45843,7 @@ var equalsFactory = (config2) => {
 };
 var containsAnyFactory = (config2) => {
   const c = config2;
-  return new DeterministicAssertionEvaluator("contains_any", (ctx) => {
+  return new DeterministicAssertionEvaluator("contains-any", (ctx) => {
     const result = runContainsAnyAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -45816,7 +45857,7 @@ var containsAnyFactory = (config2) => {
 };
 var containsAllFactory = (config2) => {
   const c = config2;
-  return new DeterministicAssertionEvaluator("contains_all", (ctx) => {
+  return new DeterministicAssertionEvaluator("contains-all", (ctx) => {
     const result = runContainsAllAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -45844,7 +45885,7 @@ var icontainsFactory = (config2) => {
 };
 var icontainsAnyFactory = (config2) => {
   const c = config2;
-  return new DeterministicAssertionEvaluator("icontains_any", (ctx) => {
+  return new DeterministicAssertionEvaluator("icontains-any", (ctx) => {
     const result = runIcontainsAnyAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -45858,7 +45899,7 @@ var icontainsAnyFactory = (config2) => {
 };
 var icontainsAllFactory = (config2) => {
   const c = config2;
-  return new DeterministicAssertionEvaluator("icontains_all", (ctx) => {
+  return new DeterministicAssertionEvaluator("icontains-all", (ctx) => {
     const result = runIcontainsAllAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -45872,7 +45913,7 @@ var icontainsAllFactory = (config2) => {
 };
 var startsWithFactory = (config2) => {
   const c = config2;
-  return new DeterministicAssertionEvaluator("starts_with", (ctx) => {
+  return new DeterministicAssertionEvaluator("starts-with", (ctx) => {
     const result = runStartsWithAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -45886,7 +45927,7 @@ var startsWithFactory = (config2) => {
 };
 var endsWithFactory = (config2) => {
   const c = config2;
-  return new DeterministicAssertionEvaluator("ends_with", (ctx) => {
+  return new DeterministicAssertionEvaluator("ends-with", (ctx) => {
     const result = runEndsWithAssertion(ctx.candidate, c.value);
     return {
       score: result.score,
@@ -45900,7 +45941,7 @@ var endsWithFactory = (config2) => {
 };
 function createBuiltinRegistry() {
   const registry2 = new EvaluatorRegistry();
-  registry2.register("llm_judge", llmJudgeFactory).register("code", codeFactory).register("composite", compositeFactory).register("tool_trajectory", toolTrajectoryFactory).register("field_accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token_usage", tokenUsageFactory).register("execution_metrics", executionMetricsFactory).register("agent_judge", agentJudgeFactory).register("contains", containsFactory).register("contains_any", containsAnyFactory).register("contains_all", containsAllFactory).register("icontains", icontainsFactory).register("icontains_any", icontainsAnyFactory).register("icontains_all", icontainsAllFactory).register("starts_with", startsWithFactory).register("ends_with", endsWithFactory).register("regex", regexFactory).register("is_json", isJsonFactory).register("equals", equalsFactory);
+  registry2.register("llm-judge", llmJudgeFactory).register("code-judge", codeFactory).register("composite", compositeFactory).register("tool-trajectory", toolTrajectoryFactory).register("field-accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token-usage", tokenUsageFactory).register("execution-metrics", executionMetricsFactory).register("agent-judge", agentJudgeFactory).register("contains", containsFactory).register("contains-any", containsAnyFactory).register("contains-all", containsAllFactory).register("icontains", icontainsFactory).register("icontains-any", icontainsAnyFactory).register("icontains-all", icontainsAllFactory).register("starts-with", startsWithFactory).register("ends-with", endsWithFactory).register("regex", regexFactory).register("is-json", isJsonFactory).register("equals", equalsFactory);
   return registry2;
 }
 async function discoverAssertions(registry2, baseDir) {
@@ -46553,7 +46594,8 @@ async function runEvaluation(options) {
     cleanupWorkspaces,
     trials,
     streamCallbacks,
-    totalBudgetUsd
+    totalBudgetUsd,
+    failOnError
   } = options;
   let useCache = options.useCache;
   if (trials && trials.count > 1 && useCache) {
@@ -46611,7 +46653,7 @@ async function runEvaluation(options) {
   };
   if (isAgentProvider(getOrCreateProvider(target)) && !target.judgeTarget) {
     throw new Error(
-      `Target "${target.name}" is an agent provider ("${target.kind}") with no judge_target \u2014 agent providers cannot return structured JSON for judging. Set judge_target to an LLM provider (e.g., azure_base).`
+      `Target "${target.name}" is an agent provider ("${target.kind}") with no judge_target \u2014 agent providers cannot return structured JSON for judging. Set judge_target to an LLM provider (e.g., azure-base).`
     );
   }
   const targetResolver = (name16) => {
@@ -46682,7 +46724,7 @@ async function runEvaluation(options) {
   const rawTemplate = suiteWorkspace?.template ?? getWorkspaceTemplate(target);
   const resolvedTemplate = await resolveWorkspaceTemplate(rawTemplate);
   const workspaceTemplate = resolvedTemplate?.dir;
-  const suiteWorkspaceFile = resolvedTemplate?.workspaceFile;
+  let suiteWorkspaceFile = resolvedTemplate?.workspaceFile;
   const isPerTestIsolation = suiteWorkspace?.isolation === "per_test";
   const hasSharedWorkspace = !!(workspaceTemplate || suiteWorkspace?.before_all || suiteWorkspace?.repos?.length && !isPerTestIsolation);
   const requestedWorkers = options.maxConcurrency ?? target.workers ?? 1;
@@ -46703,6 +46745,14 @@ async function runEvaluation(options) {
       const message = error40 instanceof Error ? error40.message : String(error40);
       throw new Error(`Failed to create shared workspace: ${message}`);
     }
+    if (suiteWorkspaceFile && sharedWorkspacePath) {
+      const copiedWorkspaceFile = path37.join(sharedWorkspacePath, path37.basename(suiteWorkspaceFile));
+      try {
+        await stat7(copiedWorkspaceFile);
+        suiteWorkspaceFile = copiedWorkspaceFile;
+      } catch {
+      }
+    }
   } else if (suiteWorkspace?.before_all || suiteWorkspace?.repos?.length && !isPerTestIsolation) {
     sharedWorkspacePath = getWorkspacePath(evalRunId, "shared");
     await mkdir12(sharedWorkspacePath, { recursive: true });
@@ -46749,6 +46799,7 @@ async function runEvaluation(options) {
   let beforeAllOutputAttached = false;
   let cumulativeBudgetCost = 0;
   let budgetExhausted = false;
+  let failOnErrorTriggered = false;
   const promises = filteredEvalCases.map(
     (evalCase) => limit(async () => {
       const workerId = nextWorkerId++;
@@ -46787,6 +46838,37 @@ async function runEvaluation(options) {
         }
         return budgetResult;
       }
+      if (failOnError === true && failOnErrorTriggered) {
+        const errorMsg = "Halted: execution error encountered with fail_on_error enabled";
+        const haltResult = {
+          timestamp: (now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
+          testId: evalCase.id,
+          dataset: evalCase.dataset,
+          score: 0,
+          hits: [],
+          misses: [],
+          answer: "",
+          target: target.name,
+          error: errorMsg,
+          executionStatus: "execution_error",
+          failureStage: "setup",
+          failureReasonCode: "error_threshold_exceeded",
+          executionError: { message: errorMsg, stage: "setup" }
+        };
+        if (onProgress) {
+          await onProgress({
+            workerId,
+            testId: evalCase.id,
+            status: "failed",
+            completedAt: Date.now(),
+            error: haltResult.error
+          });
+        }
+        if (onResult) {
+          await onResult(haltResult);
+        }
+        return haltResult;
+      }
       if (onProgress) {
         await onProgress({
           workerId,
@@ -46839,6 +46921,9 @@ async function runEvaluation(options) {
             }
           }
         }
+        if (failOnError === true && result.executionStatus === "execution_error") {
+          failOnErrorTriggered = true;
+        }
         if (beforeAllOutput && !beforeAllOutputAttached) {
           result = { ...result, beforeAllOutput };
           beforeAllOutputAttached = true;
@@ -47146,6 +47231,14 @@ async function runEvalCase(options) {
           "template_error"
         );
       }
+      if (caseWorkspaceFile && workspacePath) {
+        const copiedFile = path37.join(workspacePath, path37.basename(caseWorkspaceFile));
+        try {
+          await stat7(copiedFile);
+          caseWorkspaceFile = copiedFile;
+        } catch {
+        }
+      }
     }
     if (!workspacePath && (evalCase.workspace?.before_all || evalCase.workspace?.repos?.length) && evalRunId) {
       workspacePath = getWorkspacePath(evalRunId, evalCase.id);
@@ -47655,8 +47748,8 @@ async function runEvaluatorsForCase(options) {
       workspacePath
     });
   }
-  const evaluatorKind = evalCase.evaluator ?? "llm_judge";
-  const activeEvaluator = evaluators[evaluatorKind] ?? evaluators.llm_judge;
+  const evaluatorKind = evalCase.evaluator ?? "llm-judge";
+  const activeEvaluator = evaluators[evaluatorKind] ?? evaluators["llm-judge"];
   if (!activeEvaluator) {
     throw new Error(`No evaluator registered for kind '${evaluatorKind}'`);
   }
@@ -47739,25 +47832,24 @@ async function runEvaluatorList(options) {
     availableTargets,
     agentTimeoutMs,
     evalFileDir,
-    llmJudge: evaluatorRegistry.llm_judge,
+    llmJudge: evaluatorRegistry["llm-judge"],
     registry: typeRegistry
   };
   for (const evaluatorConfig of evaluators ?? []) {
     try {
       const evaluatorInstance = await typeRegistry.create(evaluatorConfig, dispatchContext);
       const score2 = await evaluatorInstance.evaluate(evalContext);
-      const resultType = evaluatorConfig.type === "code" ? "code_judge" : evaluatorConfig.type;
       const weight = evaluatorConfig.weight ?? 1;
       scored.push({
         score: score2,
         name: evaluatorConfig.name,
-        type: resultType,
+        type: evaluatorConfig.type,
         weight,
         ...evaluatorConfig.required !== void 0 ? { required: evaluatorConfig.required } : {}
       });
       scores.push({
         name: evaluatorConfig.name,
-        type: resultType,
+        type: evaluatorConfig.type,
         score: score2.score,
         weight,
         verdict: score2.verdict,
@@ -47779,18 +47871,17 @@ async function runEvaluatorList(options) {
         expectedAspectCount: 1,
         reasoning: message
       };
-      const resultType = evaluatorConfig.type === "code" ? "code_judge" : evaluatorConfig.type;
       const weight = evaluatorConfig.weight ?? 1;
       scored.push({
         score: fallbackScore,
         name: evaluatorConfig.name ?? "unknown",
-        type: resultType ?? "llm_judge",
+        type: evaluatorConfig.type ?? "llm-judge",
         weight,
         ...evaluatorConfig.required !== void 0 ? { required: evaluatorConfig.required } : {}
       });
       scores.push({
         name: evaluatorConfig.name ?? "unknown",
-        type: resultType ?? "llm_judge",
+        type: evaluatorConfig.type ?? "llm-judge",
         score: 0,
         weight,
         verdict: "fail",
@@ -47851,7 +47942,7 @@ function filterEvalCases(evalCases, filter2) {
   return evalCases.filter((evalCase) => micromatch4.isMatch(evalCase.id, filter2));
 }
 function buildEvaluatorRegistry(overrides, resolveJudgeProvider) {
-  const llmJudge = overrides?.llm_judge ?? new LlmJudgeEvaluator({
+  const llmJudge = overrides?.["llm-judge"] ?? new LlmJudgeEvaluator({
     resolveJudgeProvider: async (context) => {
       if (context.judgeProvider) {
         return context.judgeProvider;
@@ -47861,7 +47952,7 @@ function buildEvaluatorRegistry(overrides, resolveJudgeProvider) {
   });
   return {
     ...overrides,
-    llm_judge: llmJudge
+    "llm-judge": llmJudge
   };
 }
 async function invokeProvider(provider, options) {
@@ -48117,12 +48208,7 @@ async function evaluate(config2) {
   };
 }
 function mapAssertionType(type) {
-  switch (type) {
-    case "code_judge":
-      return "code";
-    default:
-      return type;
-  }
+  return type.replace(/_/g, "-");
 }
 function computeSummary(results, durationMs) {
   const total = results.length;
@@ -48851,6 +48937,7 @@ export {
   extractTargetsFromTestCase,
   extractTrialsConfig,
   extractCacheConfig,
+  extractFailOnError,
   detectFormat,
   buildPromptInputs,
   readTestSuiteMetadata,
@@ -48950,4 +49037,4 @@ export {
   OtelStreamingObserver,
   createAgentKernel
 };
-//# sourceMappingURL=chunk-LUHCYBMD.js.map
+//# sourceMappingURL=chunk-OQN2GDEU.js.map