@agentv/core 4.20.0 → 4.21.0-next.1

package/dist/evaluation/validation/index.cjs

@@ -260,6 +260,63 @@ async function loadCasesFromFile(filePath) {
     `Unsupported external test file format '${ext}': ${filePath}. Supported: .yaml, .yml, .jsonl`
   );
 }
+async function loadCasesFromDirectory(dirPath) {
+  const entries = await (0, import_promises2.readdir)(dirPath, { withFileTypes: true });
+  const subdirs = entries.filter((e) => e.isDirectory()).sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0);
+  const results = [];
+  for (const subdir of subdirs) {
+    const subdirPath = import_node_path2.default.join(dirPath, subdir.name);
+    let caseFilePath;
+    for (const filename of ["case.yaml", "case.yml"]) {
+      const candidate = import_node_path2.default.join(subdirPath, filename);
+      try {
+        const s = await (0, import_promises2.stat)(candidate);
+        if (s.isFile()) {
+          caseFilePath = candidate;
+          break;
+        }
+      } catch {
+      }
+    }
+    if (!caseFilePath) {
+      console.warn(
+        `${ANSI_YELLOW}Warning: Skipping directory '${subdir.name}' \u2014 no case.yaml found${ANSI_RESET}`
+      );
+      continue;
+    }
+    let content;
+    try {
+      content = await (0, import_promises2.readFile)(caseFilePath, "utf8");
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(`Cannot read case file: ${caseFilePath}
+  ${message}`);
+    }
+    const raw = (0, import_yaml2.parse)(content);
+    const parsed = interpolateEnv(raw, process.env);
+    if (!isJsonObject(parsed)) {
+      throw new Error(
+        `Case file must contain a YAML object, got ${typeof parsed}: ${caseFilePath}`
+      );
+    }
+    const caseObj = { ...parsed };
+    if (caseObj.id === void 0 || caseObj.id === null) {
+      caseObj.id = subdir.name;
+    }
+    if (!caseObj.workspace) {
+      const workspaceDirPath = import_node_path2.default.join(subdirPath, "workspace");
+      try {
+        const s = await (0, import_promises2.stat)(workspaceDirPath);
+        if (s.isDirectory()) {
+          caseObj.workspace = { template: workspaceDirPath };
+        }
+      } catch {
+      }
+    }
+    results.push(caseObj);
+  }
+  return results;
+}
 
 // src/evaluation/validation/eval-validator.ts
 var ASSERTION_TYPES_WITH_STRING_VALUE = /* @__PURE__ */ new Set([
@@ -432,17 +489,21 @@ async function validateEvalFile(filePath) {
   }
   const cases = parsed.tests;
   if (typeof cases === "string") {
-    validateTestsStringPath(cases, absolutePath, errors);
     await validateWorkspaceConfig(parsed.workspace, absolutePath, errors, "workspace");
-    const ext = import_node_path3.default.extname(cases).toLowerCase();
-    if (VALID_TEST_FILE_EXTENSIONS.has(ext)) {
-      const externalCasesPath = import_node_path3.default.resolve(import_node_path3.default.dirname(absolutePath), cases);
+    const externalCasesPath = import_node_path3.default.resolve(import_node_path3.default.dirname(absolutePath), cases);
+    let isDir = false;
+    try {
+      const pathStat = await (0, import_promises3.stat)(externalCasesPath);
+      isDir = pathStat.isDirectory();
+    } catch {
+    }
+    if (isDir) {
       try {
-        const externalCases = await loadCasesFromFile(externalCasesPath);
-        for (let i = 0; i < externalCases.length; i++) {
-          const externalCase = externalCases[i];
+        const dirCases = await loadCasesFromDirectory(externalCasesPath);
+        for (let i = 0; i < dirCases.length; i++) {
+          const dirCase = dirCases[i];
           await validateWorkspaceConfig(
-            externalCase.workspace,
+            dirCase.workspace,
             absolutePath,
             errors,
             `tests[${i}].workspace`
@@ -457,6 +518,31 @@ async function validateEvalFile(filePath) {
           message
         });
       }
+    } else {
+      validateTestsStringPath(cases, absolutePath, errors);
+      const ext = import_node_path3.default.extname(cases).toLowerCase();
+      if (VALID_TEST_FILE_EXTENSIONS.has(ext)) {
+        try {
+          const externalCases = await loadCasesFromFile(externalCasesPath);
+          for (let i = 0; i < externalCases.length; i++) {
+            const externalCase = externalCases[i];
+            await validateWorkspaceConfig(
+              externalCase.workspace,
+              absolutePath,
+              errors,
+              `tests[${i}].workspace`
+            );
+          }
+        } catch (error) {
+          const message = error instanceof Error ? error.message : String(error);
+          errors.push({
+            severity: "error",
+            filePath: absolutePath,
+            location: "tests",
+            message
+          });
+        }
+      }
     }
     return {
       valid: errors.filter((e) => e.severity === "error").length === 0,