@agentv/core 2.5.8 → 2.7.1-next.1

package/dist/evaluation/validation/index.js

@@ -4,8 +4,9 @@ import {
   PROVIDER_ALIASES,
   buildSearchRoots,
   findGitRoot,
+  isEvaluatorKind,
   resolveFileReference
-} from "../../chunk-LGQ5OPJD.js";
+} from "../../chunk-6W5E3VR6.js";
 
 // src/evaluation/validation/file-type.ts
 import { readFile } from "node:fs/promises";
@@ -73,6 +74,9 @@ function getExpectedSchema(fileType) {
 import { readFile as readFile2 } from "node:fs/promises";
 import path2 from "node:path";
 import { parse as parse2 } from "yaml";
+var ASSERTION_TYPES_WITH_VALUE = /* @__PURE__ */ new Set(["contains", "equals", "regex"]);
+var VALID_TEST_FILE_EXTENSIONS = /* @__PURE__ */ new Set([".yaml", ".yml", ".jsonl"]);
+var NAME_PATTERN = /^[a-z0-9-]+$/;
 function isObject(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
@@ -109,13 +113,41 @@ async function validateEvalFile(filePath) {
       errors
     };
   }
-  const evalcases = parsed.evalcases;
-  if (!Array.isArray(evalcases)) {
+  validateMetadata(parsed, absolutePath, errors);
+  let cases = parsed.tests;
+  if (cases === void 0 && "eval_cases" in parsed) {
+    cases = parsed.eval_cases;
     errors.push({
-      severity: "error",
+      severity: "warning",
+      filePath: absolutePath,
+      location: "eval_cases",
+      message: "'eval_cases' is deprecated. Use 'tests' instead."
+    });
+  }
+  if (cases === void 0 && "evalcases" in parsed) {
+    cases = parsed.evalcases;
+    errors.push({
+      severity: "warning",
       filePath: absolutePath,
       location: "evalcases",
-      message: "Missing or invalid 'evalcases' field (must be an array)"
+      message: "'evalcases' is deprecated. Use 'tests' instead."
+    });
+  }
+  if (typeof cases === "string") {
+    validateTestsStringPath(cases, absolutePath, errors);
+    return {
+      valid: errors.filter((e) => e.severity === "error").length === 0,
+      filePath: absolutePath,
+      fileType: "eval",
+      errors
+    };
+  }
+  if (!Array.isArray(cases)) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      location: "tests",
+      message: "Missing or invalid 'tests' field (must be an array or a file path string)"
     });
     return {
       valid: errors.length === 0,
@@ -124,9 +156,9 @@ async function validateEvalFile(filePath) {
       errors
     };
   }
-  for (let i = 0; i < evalcases.length; i++) {
-    const evalCase = evalcases[i];
-    const location = `evalcases[${i}]`;
+  for (let i = 0; i < cases.length; i++) {
+    const evalCase = cases[i];
+    const location = `tests[${i}]`;
     if (!isObject(evalCase)) {
       errors.push({
         severity: "error",
@@ -145,23 +177,29 @@ async function validateEvalFile(filePath) {
         message: "Missing or invalid 'id' field (must be a non-empty string)"
       });
     }
-    const expectedOutcome = evalCase.expected_outcome ?? evalCase.outcome;
-    if (expectedOutcome !== void 0 && (typeof expectedOutcome !== "string" || expectedOutcome.trim().length === 0)) {
+    let criteria = evalCase.criteria;
+    if (criteria === void 0 && "expected_outcome" in evalCase) {
+      criteria = evalCase.expected_outcome;
       errors.push({
-        severity: "error",
+        severity: "warning",
         filePath: absolutePath,
         location: `${location}.expected_outcome`,
-        message: "Invalid 'expected_outcome' or 'outcome' field (must be a non-empty string if provided)"
+        message: "'expected_outcome' is deprecated. Use 'criteria' instead."
       });
     }
-    const inputMessages = evalCase.input_messages;
-    const inputAlias = evalCase.input;
-    if (Array.isArray(inputMessages)) {
-      validateMessages(inputMessages, `${location}.input_messages`, absolutePath, errors);
-    } else if (inputAlias !== void 0) {
-      if (typeof inputAlias === "string") {
-      } else if (Array.isArray(inputAlias)) {
-        validateMessages(inputAlias, `${location}.input`, absolutePath, errors);
+    if (criteria !== void 0 && (typeof criteria !== "string" || criteria.trim().length === 0)) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.criteria`,
+        message: "Invalid 'criteria' field (must be a non-empty string if provided)"
+      });
+    }
+    const inputField = evalCase.input;
+    if (inputField !== void 0) {
+      if (typeof inputField === "string") {
+      } else if (Array.isArray(inputField)) {
+        validateMessages(inputField, `${location}.input`, absolutePath, errors);
       } else {
         errors.push({
           severity: "error",
@@ -174,33 +212,23 @@ async function validateEvalFile(filePath) {
       errors.push({
         severity: "error",
         filePath: absolutePath,
-        location: `${location}.input_messages`,
-        message: "Missing 'input_messages' or 'input' field (must provide one)"
+        location: `${location}.input`,
+        message: "Missing 'input' field (must be a string or array of messages)"
       });
     }
-    const expectedMessages = evalCase.expected_messages;
-    const expectedOutputAlias = evalCase.expected_output;
-    if (expectedMessages !== void 0 && !Array.isArray(expectedMessages)) {
-      errors.push({
-        severity: "error",
-        filePath: absolutePath,
-        location: `${location}.expected_messages`,
-        message: "Invalid 'expected_messages' field (must be an array if provided)"
-      });
-    } else if (Array.isArray(expectedMessages)) {
-      validateMessages(expectedMessages, `${location}.expected_messages`, absolutePath, errors);
-    } else if (expectedOutputAlias !== void 0) {
-      if (typeof expectedOutputAlias === "string") {
-      } else if (Array.isArray(expectedOutputAlias)) {
-        if (expectedOutputAlias.length > 0 && isObject(expectedOutputAlias[0]) && "role" in expectedOutputAlias[0]) {
+    const expectedOutputField = evalCase.expected_output;
+    if (expectedOutputField !== void 0) {
+      if (typeof expectedOutputField === "string") {
+      } else if (Array.isArray(expectedOutputField)) {
+        if (expectedOutputField.length > 0 && isObject(expectedOutputField[0]) && "role" in expectedOutputField[0]) {
           validateMessages(
-            expectedOutputAlias,
+            expectedOutputField,
             `${location}.expected_output`,
             absolutePath,
             errors
           );
         }
-      } else if (isObject(expectedOutputAlias)) {
+      } else if (isObject(expectedOutputField)) {
       } else {
         errors.push({
           severity: "error",
@@ -210,9 +238,13 @@ async function validateEvalFile(filePath) {
         });
       }
     }
+    const assertField = evalCase.assert;
+    if (assertField !== void 0) {
+      validateAssertArray(assertField, location, absolutePath, errors);
+    }
   }
   return {
-    valid: errors.length === 0,
+    valid: errors.filter((e) => e.severity === "error").length === 0,
     filePath: absolutePath,
     fileType: "eval",
     errors
@@ -292,6 +324,133 @@ function validateMessages(messages, location, filePath, errors) {
     }
   }
 }
+function validateMetadata(parsed, filePath, errors) {
+  const name = parsed.name;
+  if (name !== void 0) {
+    if (typeof name === "string") {
+      if (!NAME_PATTERN.test(name)) {
+        errors.push({
+          severity: "warning",
+          filePath,
+          location: "name",
+          message: `Invalid 'name' format '${name}'. Must match pattern /^[a-z0-9-]+$/ (lowercase alphanumeric with hyphens).`
+        });
+      }
+    }
+    if (!("description" in parsed) || parsed.description === void 0) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location: "name",
+        message: "When 'name' is present, 'description' should also be provided."
+      });
+    }
+  }
+}
+function validateTestsStringPath(testsPath, filePath, errors) {
+  const ext = path2.extname(testsPath);
+  if (!VALID_TEST_FILE_EXTENSIONS.has(ext)) {
+    errors.push({
+      severity: "warning",
+      filePath,
+      location: "tests",
+      message: `Unsupported file extension '${ext}' for tests path '${testsPath}'. Supported extensions: ${[...VALID_TEST_FILE_EXTENSIONS].join(", ")}`
+    });
+  }
+}
+function validateAssertArray(assertField, parentLocation, filePath, errors) {
+  if (!Array.isArray(assertField)) {
+    errors.push({
+      severity: "warning",
+      filePath,
+      location: `${parentLocation}.assert`,
+      message: "'assert' must be an array of assertion objects."
+    });
+    return;
+  }
+  for (let i = 0; i < assertField.length; i++) {
+    const item = assertField[i];
+    const location = `${parentLocation}.assert[${i}]`;
+    if (!isObject(item)) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location,
+        message: "Assertion item must be an object with a type field."
+      });
+      continue;
+    }
+    const typeValue = item.type;
+    if (typeValue === void 0 || typeof typeValue !== "string") {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location: `${location}.type`,
+        message: "Assertion item is missing a 'type' field."
+      });
+      continue;
+    }
+    if (!isEvaluatorKind(typeValue)) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location: `${location}.type`,
+        message: `Unknown assertion type '${typeValue}'.`
+      });
+      continue;
+    }
+    if (ASSERTION_TYPES_WITH_VALUE.has(typeValue)) {
+      const value = item.value;
+      if (value === void 0 || typeof value !== "string") {
+        errors.push({
+          severity: "warning",
+          filePath,
+          location: `${location}.value`,
+          message: `Assertion type '${typeValue}' requires a 'value' field (string).`
+        });
+        continue;
+      }
+      if (typeValue === "regex") {
+        try {
+          new RegExp(value);
+        } catch {
+          errors.push({
+            severity: "warning",
+            filePath,
+            location: `${location}.value`,
+            message: `Invalid regex pattern '${value}': not a valid regular expression.`
+          });
+        }
+      }
+    }
+    const required = item.required;
+    if (required !== void 0) {
+      validateRequiredField(required, location, filePath, errors);
+    }
+  }
+}
+function validateRequiredField(required, parentLocation, filePath, errors) {
+  if (typeof required === "boolean") {
+    return;
+  }
+  if (typeof required === "number") {
+    if (required <= 0 || required > 1) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location: `${parentLocation}.required`,
+        message: `Invalid 'required' value ${required}. When a number, it must be between 0 (exclusive) and 1 (inclusive).`
+      });
+    }
+    return;
+  }
+  errors.push({
+    severity: "warning",
+    filePath,
+    location: `${parentLocation}.required`,
+    message: `Invalid 'required' value. Must be a boolean or a number between 0 (exclusive) and 1 (inclusive).`
+  });
+}
 function validateContentForRoleMarkers(content, location, filePath, errors) {
   const markers = ["@[System]:", "@[User]:", "@[Assistant]:", "@[Tool]:"];
   for (const marker of markers) {
@@ -371,6 +530,7 @@ var GEMINI_SETTINGS = /* @__PURE__ */ new Set([
 ]);
 var CODEX_SETTINGS = /* @__PURE__ */ new Set([
   ...COMMON_SETTINGS,
+  "model",
   "executable",
   "command",
   "binary",
@@ -386,41 +546,45 @@ var CODEX_SETTINGS = /* @__PURE__ */ new Set([
   "log_format",
   "logFormat",
   "log_output_format",
-  "logOutputFormat"
+  "logOutputFormat",
+  "system_prompt",
+  "systemPrompt",
+  "workspace_template",
+  "workspaceTemplate"
 ]);
-var COPILOT_SETTINGS = /* @__PURE__ */ new Set([
+var COPILOT_SDK_SETTINGS = /* @__PURE__ */ new Set([
   ...COMMON_SETTINGS,
-  "executable",
-  "command",
-  "binary",
-  "args",
-  "arguments",
+  "cli_url",
+  "cliUrl",
+  "cli_path",
+  "cliPath",
+  "github_token",
+  "githubToken",
   "model",
   "cwd",
   "timeout_seconds",
   "timeoutSeconds",
   "log_dir",
   "logDir",
-  "log_directory",
-  "logDirectory",
   "log_format",
   "logFormat",
-  "log_output_format",
-  "logOutputFormat",
   "system_prompt",
-  "systemPrompt"
+  "systemPrompt",
+  "workspace_template",
+  "workspaceTemplate"
 ]);
 var VSCODE_SETTINGS = /* @__PURE__ */ new Set([
   ...COMMON_SETTINGS,
+  "executable",
   "workspace_template",
   "workspaceTemplate",
-  "vscode_cmd",
-  "command",
   "wait",
   "dry_run",
   "dryRun",
   "subagent_root",
-  "subagentRoot"
+  "subagentRoot",
+  "timeout_seconds",
+  "timeoutSeconds"
 ]);
 var MOCK_SETTINGS = /* @__PURE__ */ new Set([
   ...COMMON_SETTINGS,
@@ -431,6 +595,29 @@ var MOCK_SETTINGS = /* @__PURE__ */ new Set([
   "trace"
   // For testing tool_trajectory evaluator
 ]);
+var CLAUDE_SETTINGS = /* @__PURE__ */ new Set([
+  ...COMMON_SETTINGS,
+  "model",
+  "cwd",
+  "timeout_seconds",
+  "timeoutSeconds",
+  "log_dir",
+  "logDir",
+  "log_directory",
+  "logDirectory",
+  "log_format",
+  "logFormat",
+  "log_output_format",
+  "logOutputFormat",
+  "system_prompt",
+  "systemPrompt",
+  "workspace_template",
+  "workspaceTemplate",
+  "max_turns",
+  "maxTurns",
+  "max_budget_usd",
+  "maxBudgetUsd"
+]);
 function getKnownSettings(provider) {
   const normalizedProvider = provider.toLowerCase();
   switch (normalizedProvider) {
@@ -446,8 +633,15 @@ function getKnownSettings(provider) {
     case "codex":
     case "codex-cli":
       return CODEX_SETTINGS;
+    case "copilot":
+    case "copilot-sdk":
+    case "copilot_sdk":
     case "copilot-cli":
-      return COPILOT_SETTINGS;
+      return COPILOT_SDK_SETTINGS;
+    case "claude":
+    case "claude-code":
+    case "claude-sdk":
+      return CLAUDE_SETTINGS;
     case "vscode":
     case "vscode-insiders":
       return VSCODE_SETTINGS;
@@ -745,7 +939,32 @@ async function validateConfigFile(filePath) {
         });
       }
     }
-    const allowedFields = /* @__PURE__ */ new Set(["$schema", "guideline_patterns"]);
+    const evalPatterns = config.eval_patterns;
+    if (evalPatterns !== void 0) {
+      if (!Array.isArray(evalPatterns)) {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: "eval_patterns",
+          message: "Field 'eval_patterns' must be an array"
+        });
+      } else if (!evalPatterns.every((p) => typeof p === "string")) {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: "eval_patterns",
+          message: "All entries in 'eval_patterns' must be strings"
+        });
+      } else if (evalPatterns.length === 0) {
+        errors.push({
+          severity: "warning",
+          filePath,
+          location: "eval_patterns",
+          message: "Field 'eval_patterns' is empty. Consider removing it or adding patterns."
+        });
+      }
+    }
+    const allowedFields = /* @__PURE__ */ new Set(["$schema", "guideline_patterns", "eval_patterns"]);
     const unexpectedFields = Object.keys(config).filter((key) => !allowedFields.has(key));
     if (unexpectedFields.length > 0) {
       errors.push({
@@ -800,30 +1019,36 @@ async function validateFileReferences(evalFilePath) {
   if (!isObject3(parsed)) {
     return errors;
   }
-  const evalcases = parsed.evalcases;
-  if (!Array.isArray(evalcases)) {
+  let cases = parsed.tests;
+  if (cases === void 0 && "eval_cases" in parsed) {
+    cases = parsed.eval_cases;
+  }
+  if (cases === void 0 && "evalcases" in parsed) {
+    cases = parsed.evalcases;
+  }
+  if (!Array.isArray(cases)) {
     return errors;
   }
-  for (let i = 0; i < evalcases.length; i++) {
-    const evalCase = evalcases[i];
+  for (let i = 0; i < cases.length; i++) {
+    const evalCase = cases[i];
     if (!isObject3(evalCase)) {
       continue;
     }
-    const inputMessages = evalCase.input_messages;
-    if (Array.isArray(inputMessages)) {
+    const inputField = evalCase.input;
+    if (Array.isArray(inputField)) {
       await validateMessagesFileRefs(
-        inputMessages,
-        `evalcases[${i}].input_messages`,
+        inputField,
+        `tests[${i}].input`,
         searchRoots,
         absolutePath,
         errors
       );
     }
-    const expectedMessages = evalCase.expected_messages;
-    if (Array.isArray(expectedMessages)) {
+    const expectedOutputField = evalCase.expected_output;
+    if (Array.isArray(expectedOutputField)) {
       await validateMessagesFileRefs(
-        expectedMessages,
-        `evalcases[${i}].expected_messages`,
+        expectedOutputField,
+        `tests[${i}].expected_output`,
         searchRoots,
         absolutePath,
         errors