@agentv/core 4.15.8-next.1 → 4.15.9-next.1

package/dist/evaluation/validation/index.cjs

@@ -33,10 +33,12 @@ __export(validation_exports, {
   detectFileType: () => detectFileType,
   getExpectedSchema: () => getExpectedSchema,
   isValidSchema: () => isValidSchema,
+  validateCasesFile: () => validateCasesFile,
   validateConfigFile: () => validateConfigFile,
   validateEvalFile: () => validateEvalFile,
   validateFileReferences: () => validateFileReferences,
-  validateTargetsFile: () => validateTargetsFile
+  validateTargetsFile: () => validateTargetsFile,
+  validateWorkspacePaths: () => validateWorkspacePaths
 });
 module.exports = __toCommonJS(validation_exports);
 
@@ -51,6 +53,9 @@ async function detectFileType(filePath) {
   try {
     const content = await (0, import_promises.readFile)(filePath, "utf8");
     const parsed = (0, import_yaml.parse)(content);
+    if (Array.isArray(parsed)) {
+      return "cases";
+    }
     if (typeof parsed !== "object" || parsed === null) {
       return inferFileTypeFromPath(filePath);
     }
@@ -84,7 +89,11 @@ function inferFileTypeFromPath(filePath) {
       return "targets";
     }
   }
-  return "eval";
+  const lower = basename.toLowerCase();
+  if (lower.endsWith(".eval.yaml") || lower.endsWith(".eval.yml")) {
+    return "eval";
+  }
+  return "unknown";
 }
 function isValidSchema(schema) {
   return schema === SCHEMA_EVAL_V2 || schema === SCHEMA_TARGETS_V2 || schema === SCHEMA_CONFIG_V2;
@@ -272,6 +281,7 @@ var KNOWN_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
   "$schema",
   "name",
   "description",
+  "category",
   "version",
   "author",
   "tags",
@@ -280,13 +290,19 @@ var KNOWN_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
   "input",
   "input_files",
   "tests",
-  "eval_cases",
   "target",
   "execution",
   "assertions",
   "evaluators",
+  "preprocessors",
   "workspace"
 ]);
+var DEPRECATED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Map([
+  ["eval_cases", "'eval_cases' is deprecated. Use 'tests' instead."],
+  ["evalcases", "'evalcases' is deprecated. Use 'tests' instead."],
+  ["evaluator", "'evaluator' is deprecated. Use 'assertions' instead."],
+  ["assert", "'assert' is deprecated. Use 'assertions' instead."]
+]);
 var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
   "id",
   "criteria",
@@ -295,12 +311,12 @@ var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
   "expected_output",
   "assertions",
   "evaluators",
+  "rubrics",
   "execution",
   "workspace",
   "metadata",
   "conversation_id",
   "suite",
-  "note",
   "depends_on",
   "on_dependency_failure",
   "mode",
@@ -309,13 +325,48 @@ var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
   "on_turn_failure",
   "window_size"
 ]);
+var DEPRECATED_TEST_FIELDS = /* @__PURE__ */ new Map([
+  ["evaluator", "'evaluator' is deprecated. Use 'assertions' instead."],
+  ["assert", "'assert' is deprecated. Use 'assertions' instead."],
+  ["expected_outcome", "'expected_outcome' is deprecated. Use 'criteria' instead."]
+]);
 var NAME_PATTERN = /^[a-z0-9-]+$/;
+var ASSERTION_SCRIPT_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".js", ".mts", ".mjs", ".cts", ".cjs"]);
+var customAssertionCache = /* @__PURE__ */ new Map();
+function discoverCustomAssertionTypes(baseDir) {
+  const resolved = import_node_path3.default.resolve(baseDir);
+  const cached = customAssertionCache.get(resolved);
+  if (cached) return cached;
+  const promise = (async () => {
+    const types = /* @__PURE__ */ new Set();
+    let dir = resolved;
+    const root = import_node_path3.default.parse(dir).root;
+    while (dir !== root) {
+      const assertionsDir = import_node_path3.default.join(dir, ".agentv", "assertions");
+      try {
+        const entries = await (0, import_promises3.readdir)(assertionsDir, { withFileTypes: true });
+        for (const entry of entries) {
+          if (!entry.isFile()) continue;
+          const ext = import_node_path3.default.extname(entry.name).toLowerCase();
+          if (!ASSERTION_SCRIPT_EXTENSIONS.has(ext)) continue;
+          types.add(entry.name.slice(0, -ext.length));
+        }
+      } catch {
+      }
+      dir = import_node_path3.default.dirname(dir);
+    }
+    return types;
+  })();
+  customAssertionCache.set(resolved, promise);
+  return promise;
+}
 function isObject(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 async function validateEvalFile(filePath) {
   const errors = [];
   const absolutePath = import_node_path3.default.resolve(filePath);
+  const customAssertionTypes = await discoverCustomAssertionTypes(import_node_path3.default.dirname(absolutePath));
   let parsed;
   try {
     const content = await (0, import_promises3.readFile)(absolutePath, "utf8");
@@ -348,7 +399,15 @@ async function validateEvalFile(filePath) {
   }
   validateMetadata(parsed, absolutePath, errors);
   for (const key of Object.keys(parsed)) {
-    if (!KNOWN_TOP_LEVEL_FIELDS.has(key)) {
+    const deprecationMessage = DEPRECATED_TOP_LEVEL_FIELDS.get(key);
+    if (deprecationMessage) {
+      errors.push({
+        severity: "warning",
+        filePath: absolutePath,
+        location: key,
+        message: deprecationMessage
+      });
+    } else if (!KNOWN_TOP_LEVEL_FIELDS.has(key)) {
       errors.push({
         severity: "warning",
         filePath: absolutePath,
@@ -446,7 +505,15 @@ async function validateEvalFile(filePath) {
       continue;
     }
     for (const key of Object.keys(evalCase)) {
-      if (!KNOWN_TEST_FIELDS.has(key)) {
+      const deprecationMessage = DEPRECATED_TEST_FIELDS.get(key);
+      if (deprecationMessage) {
+        errors.push({
+          severity: "warning",
+          filePath: absolutePath,
+          location: `${location}.${key}`,
+          message: deprecationMessage
+        });
+      } else if (!KNOWN_TEST_FIELDS.has(key)) {
         errors.push({
           severity: "warning",
           filePath: absolutePath,
@@ -518,7 +585,7 @@ async function validateEvalFile(filePath) {
     }
     const assertField = evalCase.assertions;
     if (assertField !== void 0) {
-      validateAssertArray(assertField, location, absolutePath, errors);
+      validateAssertArray(assertField, location, absolutePath, errors, customAssertionTypes);
     }
     validateConversationMode(evalCase, location, absolutePath, errors);
     await validateWorkspaceConfig(
@@ -729,7 +796,7 @@ function validateTestsStringPath(testsPath, filePath, errors) {
     });
   }
 }
-function validateAssertArray(assertField, parentLocation, filePath, errors) {
+function validateAssertArray(assertField, parentLocation, filePath, errors, customAssertionTypes = /* @__PURE__ */ new Set()) {
   if (!Array.isArray(assertField)) {
     errors.push({
       severity: "warning",
@@ -777,7 +844,7 @@ function validateAssertArray(assertField, parentLocation, filePath, errors) {
       continue;
     }
     const typeValue = rawTypeValue.replace(/_/g, "-");
-    if (!isEvaluatorKind(typeValue)) {
+    if (!isEvaluatorKind(typeValue) && !customAssertionTypes.has(typeValue)) {
       errors.push({
         severity: "warning",
         filePath,
@@ -944,13 +1011,89 @@ function validateConversationMode(evalCase, location, filePath, errors) {
   }
 }
 
-// src/evaluation/validation/targets-validator.ts
+// src/evaluation/validation/cases-validator.ts
 var import_promises4 = require("fs/promises");
-var import_node_path5 = __toESM(require("path"), 1);
+var import_node_path4 = __toESM(require("path"), 1);
 var import_yaml4 = require("yaml");
+function isObject2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+async function validateCasesFile(filePath) {
+  const errors = [];
+  const absolutePath = import_node_path4.default.resolve(filePath);
+  let parsed;
+  try {
+    const content = await (0, import_promises4.readFile)(absolutePath, "utf8");
+    parsed = (0, import_yaml4.parse)(content);
+  } catch (error) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      message: `Failed to parse YAML: ${error.message}`
+    });
+    return { valid: false, filePath: absolutePath, fileType: "cases", errors };
+  }
+  if (!Array.isArray(parsed)) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      message: "Cases file must contain a YAML array of test case objects"
+    });
+    return { valid: false, filePath: absolutePath, fileType: "cases", errors };
+  }
+  for (let i = 0; i < parsed.length; i++) {
+    const item = parsed[i];
+    const location = `[${i}]`;
+    if (!isObject2(item)) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location,
+        message: "Each test case must be an object"
+      });
+      continue;
+    }
+    const id = item.id;
+    if (typeof id !== "string" || id.trim().length === 0) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.id`,
+        message: "Missing or invalid 'id' field (must be a non-empty string)"
+      });
+    }
+    const input = item.input;
+    if (input === void 0) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.input`,
+        message: "Missing 'input' field (must be a string or array of messages)"
+      });
+    } else if (typeof input !== "string" && !Array.isArray(input)) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.input`,
+        message: "Invalid 'input' field (must be a string or array of messages)"
+      });
+    }
+  }
+  return {
+    valid: errors.filter((e) => e.severity === "error").length === 0,
+    filePath: absolutePath,
+    fileType: "cases",
+    errors
+  };
+}
+
+// src/evaluation/validation/targets-validator.ts
+var import_promises5 = require("fs/promises");
+var import_node_path6 = __toESM(require("path"), 1);
+var import_yaml5 = require("yaml");
 
 // src/evaluation/providers/targets.ts
-var import_node_path4 = __toESM(require("path"), 1);
+var import_node_path5 = __toESM(require("path"), 1);
 var import_zod = require("zod");
 var CliHealthcheckHttpInputSchema = import_zod.z.object({
   url: import_zod.z.string().min(1, "healthcheck URL is required"),
@@ -1163,7 +1306,7 @@ var PROVIDER_ALIASES = [
 ];
 
 // src/evaluation/validation/targets-validator.ts
-function isObject2(value) {
+function isObject3(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 var COMMON_SETTINGS = new Set(COMMON_TARGET_SETTINGS);
@@ -1385,11 +1528,11 @@ function validateUnknownSettings(target, provider, absolutePath, location, error
 }
 async function validateTargetsFile(filePath) {
   const errors = [];
-  const absolutePath = import_node_path5.default.resolve(filePath);
+  const absolutePath = import_node_path6.default.resolve(filePath);
   let parsed;
   try {
-    const content = await (0, import_promises4.readFile)(absolutePath, "utf8");
-    parsed = (0, import_yaml4.parse)(content);
+    const content = await (0, import_promises5.readFile)(absolutePath, "utf8");
+    parsed = (0, import_yaml5.parse)(content);
   } catch (error) {
     errors.push({
       severity: "error",
@@ -1421,7 +1564,7 @@ async function validateTargetsFile(filePath) {
     }
   }
   function validateCliHealthcheck(healthcheck, absolutePath2, location, errors2) {
-    if (!isObject2(healthcheck)) {
+    if (!isObject3(healthcheck)) {
       errors2.push({
         severity: "error",
         filePath: absolutePath2,
@@ -1496,7 +1639,7 @@ async function validateTargetsFile(filePath) {
     }
     return result;
   }
-  if (!isObject2(parsed)) {
+  if (!isObject3(parsed)) {
     errors.push({
       severity: "error",
       filePath: absolutePath,
@@ -1528,7 +1671,7 @@ async function validateTargetsFile(filePath) {
   for (let i = 0; i < targets.length; i++) {
     const target = targets[i];
     const location = `targets[${i}]`;
-    if (!isObject2(target)) {
+    if (!isObject3(target)) {
       errors.push({
         severity: "error",
         filePath: absolutePath,
@@ -1602,13 +1745,13 @@ async function validateTargetsFile(filePath) {
 }
 
 // src/evaluation/validation/config-validator.ts
-var import_promises5 = require("fs/promises");
-var import_yaml5 = require("yaml");
+var import_promises6 = require("fs/promises");
+var import_yaml6 = require("yaml");
 async function validateConfigFile(filePath) {
   const errors = [];
   try {
-    const content = await (0, import_promises5.readFile)(filePath, "utf8");
-    const parsed = (0, import_yaml5.parse)(content);
+    const content = await (0, import_promises6.readFile)(filePath, "utf8");
+    const parsed = (0, import_yaml6.parse)(content);
     if (typeof parsed !== "object" || parsed === null) {
       errors.push({
         severity: "error",
@@ -1744,31 +1887,31 @@ async function validateConfigFile(filePath) {
 }
 
 // src/evaluation/validation/file-reference-validator.ts
-var import_promises7 = require("fs/promises");
-var import_node_path7 = __toESM(require("path"), 1);
-var import_yaml6 = require("yaml");
+var import_promises8 = require("fs/promises");
+var import_node_path8 = __toESM(require("path"), 1);
+var import_yaml7 = require("yaml");
 
 // src/evaluation/file-utils.ts
 var import_node_fs = require("fs");
-var import_promises6 = require("fs/promises");
-var import_node_path6 = __toESM(require("path"), 1);
+var import_promises7 = require("fs/promises");
+var import_node_path7 = __toESM(require("path"), 1);
 async function fileExists(filePath) {
   try {
-    await (0, import_promises6.access)(filePath, import_node_fs.constants.F_OK);
+    await (0, import_promises7.access)(filePath, import_node_fs.constants.F_OK);
     return true;
   } catch {
     return false;
   }
 }
 async function findGitRoot(startPath) {
-  let currentDir = import_node_path6.default.dirname(import_node_path6.default.resolve(startPath));
-  const root = import_node_path6.default.parse(currentDir).root;
+  let currentDir = import_node_path7.default.dirname(import_node_path7.default.resolve(startPath));
+  const root = import_node_path7.default.parse(currentDir).root;
   while (currentDir !== root) {
-    const gitPath = import_node_path6.default.join(currentDir, ".git");
+    const gitPath = import_node_path7.default.join(currentDir, ".git");
     if (await fileExists(gitPath)) {
       return currentDir;
     }
-    const parentDir = import_node_path6.default.dirname(currentDir);
+    const parentDir = import_node_path7.default.dirname(currentDir);
     if (parentDir === currentDir) {
       break;
     }
@@ -1779,16 +1922,16 @@ async function findGitRoot(startPath) {
 function buildSearchRoots(evalPath, repoRoot) {
   const uniqueRoots = [];
   const addRoot = (root) => {
-    const normalized = import_node_path6.default.resolve(root);
+    const normalized = import_node_path7.default.resolve(root);
     if (!uniqueRoots.includes(normalized)) {
       uniqueRoots.push(normalized);
     }
   };
-  let currentDir = import_node_path6.default.dirname(evalPath);
+  let currentDir = import_node_path7.default.dirname(evalPath);
   let reachedBoundary = false;
   while (!reachedBoundary) {
     addRoot(currentDir);
-    const parentDir = import_node_path6.default.dirname(currentDir);
+    const parentDir = import_node_path7.default.dirname(currentDir);
     if (currentDir === repoRoot || parentDir === currentDir) {
       reachedBoundary = true;
     } else {
@@ -1806,16 +1949,16 @@ function trimLeadingSeparators(value) {
 async function resolveFileReference(rawValue, searchRoots) {
   const displayPath = trimLeadingSeparators(rawValue);
   const potentialPaths = [];
-  if (import_node_path6.default.isAbsolute(rawValue)) {
-    potentialPaths.push(import_node_path6.default.normalize(rawValue));
+  if (import_node_path7.default.isAbsolute(rawValue)) {
+    potentialPaths.push(import_node_path7.default.normalize(rawValue));
   }
   for (const base of searchRoots) {
-    potentialPaths.push(import_node_path6.default.resolve(base, displayPath));
+    potentialPaths.push(import_node_path7.default.resolve(base, displayPath));
   }
   const attempted = [];
   const seen = /* @__PURE__ */ new Set();
   for (const candidate of potentialPaths) {
-    const absoluteCandidate = import_node_path6.default.resolve(candidate);
+    const absoluteCandidate = import_node_path7.default.resolve(candidate);
     if (seen.has(absoluteCandidate)) {
       continue;
     }
@@ -1829,12 +1972,12 @@ async function resolveFileReference(rawValue, searchRoots) {
 }
 
 // src/evaluation/validation/file-reference-validator.ts
-function isObject3(value) {
+function isObject4(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 async function validateFileReferences(evalFilePath) {
   const errors = [];
-  const absolutePath = import_node_path7.default.resolve(evalFilePath);
+  const absolutePath = import_node_path8.default.resolve(evalFilePath);
   const gitRoot = await findGitRoot(absolutePath);
   if (!gitRoot) {
     errors.push({
@@ -1847,12 +1990,12 @@ async function validateFileReferences(evalFilePath) {
   const searchRoots = buildSearchRoots(absolutePath, gitRoot);
   let parsed;
   try {
-    const content = await (0, import_promises7.readFile)(absolutePath, "utf8");
-    parsed = (0, import_yaml6.parse)(content);
+    const content = await (0, import_promises8.readFile)(absolutePath, "utf8");
+    parsed = (0, import_yaml7.parse)(content);
   } catch {
     return errors;
   }
-  if (!isObject3(parsed)) {
+  if (!isObject4(parsed)) {
     return errors;
   }
   let cases = parsed.tests;
@@ -1867,7 +2010,7 @@ async function validateFileReferences(evalFilePath) {
   }
   for (let i = 0; i < cases.length; i++) {
     const evalCase = cases[i];
-    if (!isObject3(evalCase)) {
+    if (!isObject4(evalCase)) {
       continue;
     }
     const inputField = evalCase.input;
@@ -1896,7 +2039,7 @@ async function validateFileReferences(evalFilePath) {
 async function validateMessagesFileRefs(messages, location, searchRoots, filePath, errors) {
   for (let i = 0; i < messages.length; i++) {
     const message = messages[i];
-    if (!isObject3(message)) {
+    if (!isObject4(message)) {
       continue;
     }
     const content = message.content;
@@ -1908,7 +2051,7 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
     }
     for (let j = 0; j < content.length; j++) {
       const contentItem = content[j];
-      if (!isObject3(contentItem)) {
+      if (!isObject4(contentItem)) {
         continue;
       }
       const type = contentItem.type;
@@ -1935,7 +2078,7 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
         });
       } else {
         try {
-          const fileContent = await (0, import_promises7.readFile)(resolvedPath, "utf8");
+          const fileContent = await (0, import_promises8.readFile)(resolvedPath, "utf8");
           if (fileContent.trim().length === 0) {
             errors.push({
               severity: "warning",
@@ -1956,14 +2099,116 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
     }
   }
 }
+
+// src/evaluation/validation/workspace-path-validator.ts
+var import_promises9 = require("fs/promises");
+var import_node_path9 = __toESM(require("path"), 1);
+var import_yaml8 = require("yaml");
+function isObject5(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+async function validateWorkspacePaths(evalFilePath) {
+  const errors = [];
+  const absolutePath = import_node_path9.default.resolve(evalFilePath);
+  const evalDir = import_node_path9.default.dirname(absolutePath);
+  let parsed;
+  try {
+    const content = await (0, import_promises9.readFile)(absolutePath, "utf8");
+    parsed = (0, import_yaml8.parse)(content);
+  } catch {
+    return errors;
+  }
+  if (!isObject5(parsed)) return errors;
+  const workspaceRaw = parsed.workspace;
+  if (workspaceRaw === void 0 || workspaceRaw === null) return errors;
+  if (typeof workspaceRaw === "string") {
+    const workspaceFilePath = import_node_path9.default.resolve(evalDir, workspaceRaw);
+    try {
+      const wsContent = await (0, import_promises9.readFile)(workspaceFilePath, "utf8");
+      const wsParsed = (0, import_yaml8.parse)(wsContent);
+      if (isObject5(wsParsed)) {
+        const wsDir = import_node_path9.default.dirname(workspaceFilePath);
+        await validateWorkspaceObject(wsParsed, wsDir, absolutePath, "workspace", errors);
+      }
+    } catch {
+    }
+  } else if (isObject5(workspaceRaw)) {
+    await validateWorkspaceObject(workspaceRaw, evalDir, absolutePath, "workspace", errors);
+  }
+  return errors;
+}
+async function validateWorkspaceObject(obj, baseDir, evalFilePath, location, errors) {
+  const template = obj.template;
+  if (typeof template === "string") {
+    const templatePath = import_node_path9.default.isAbsolute(template) ? template : import_node_path9.default.resolve(baseDir, template);
+    if (!await fileExists2(templatePath)) {
+      errors.push({
+        severity: "error",
+        filePath: evalFilePath,
+        location: `${location}.template`,
+        message: `Template path not found: ${template} (resolved to ${templatePath})`
+      });
+    }
+  }
+  const hooks = obj.hooks;
+  if (!isObject5(hooks)) return;
+  for (const hookName of ["before_all", "before_each", "after_each", "after_all"]) {
+    const hook = hooks[hookName];
+    if (!isObject5(hook)) continue;
+    const hookCwdRaw = typeof hook.cwd === "string" ? hook.cwd : void 0;
+    const hookCwd = hookCwdRaw ? import_node_path9.default.isAbsolute(hookCwdRaw) ? hookCwdRaw : import_node_path9.default.resolve(baseDir, hookCwdRaw) : baseDir;
+    const command = hook.command ?? hook.script;
+    if (!Array.isArray(command)) continue;
+    for (let i = 0; i < command.length; i++) {
+      const arg = command[i];
+      if (typeof arg !== "string") continue;
+      if (!looksLikeFilePath(arg)) continue;
+      const resolved = import_node_path9.default.isAbsolute(arg) ? arg : import_node_path9.default.resolve(hookCwd, arg);
+      if (!await fileExists2(resolved)) {
+        errors.push({
+          severity: "error",
+          filePath: evalFilePath,
+          location: `${location}.hooks.${hookName}.command[${i}]`,
+          message: `Hook script not found: ${arg} (resolved to ${resolved})`
+        });
+      }
+    }
+  }
+}
+function looksLikeFilePath(arg) {
+  if (arg.startsWith("./") || arg.startsWith("../")) return true;
+  const scriptExtensions = [
+    ".mjs",
+    ".cjs",
+    ".js",
+    ".ts",
+    ".sh",
+    ".bash",
+    ".zsh",
+    ".py",
+    ".rb",
+    ".pl"
+  ];
+  return scriptExtensions.some((ext) => arg.endsWith(ext));
+}
+async function fileExists2(filePath) {
+  try {
+    await (0, import_promises9.access)(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   detectFileType,
   getExpectedSchema,
   isValidSchema,
+  validateCasesFile,
   validateConfigFile,
   validateEvalFile,
   validateFileReferences,
-  validateTargetsFile
+  validateTargetsFile,
+  validateWorkspacePaths
 });
 //# sourceMappingURL=index.cjs.map