@agentv/core 4.20.0 → 4.21.0-next.1

package/dist/index.cjs

@@ -559,6 +559,63 @@ async function resolveFileReference(ref, evalFileDir) {
   }
   return loadCasesFromFile(absolutePattern);
 }
+async function loadCasesFromDirectory(dirPath) {
+  const entries = await (0, import_promises2.readdir)(dirPath, { withFileTypes: true });
+  const subdirs = entries.filter((e) => e.isDirectory()).sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0);
+  const results = [];
+  for (const subdir of subdirs) {
+    const subdirPath = import_node_path2.default.join(dirPath, subdir.name);
+    let caseFilePath;
+    for (const filename of ["case.yaml", "case.yml"]) {
+      const candidate = import_node_path2.default.join(subdirPath, filename);
+      try {
+        const s = await (0, import_promises2.stat)(candidate);
+        if (s.isFile()) {
+          caseFilePath = candidate;
+          break;
+        }
+      } catch {
+      }
+    }
+    if (!caseFilePath) {
+      console.warn(
+        `${ANSI_YELLOW}Warning: Skipping directory '${subdir.name}' \u2014 no case.yaml found${ANSI_RESET2}`
+      );
+      continue;
+    }
+    let content;
+    try {
+      content = await (0, import_promises2.readFile)(caseFilePath, "utf8");
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(`Cannot read case file: ${caseFilePath}
+  ${message}`);
+    }
+    const raw = (0, import_yaml.parse)(content);
+    const parsed = interpolateEnv(raw, process.env);
+    if (!isJsonObject(parsed)) {
+      throw new Error(
+        `Case file must contain a YAML object, got ${typeof parsed}: ${caseFilePath}`
+      );
+    }
+    const caseObj = { ...parsed };
+    if (caseObj.id === void 0 || caseObj.id === null) {
+      caseObj.id = subdir.name;
+    }
+    if (!caseObj.workspace) {
+      const workspaceDirPath = import_node_path2.default.join(subdirPath, "workspace");
+      try {
+        const s = await (0, import_promises2.stat)(workspaceDirPath);
+        if (s.isDirectory()) {
+          caseObj.workspace = { template: workspaceDirPath };
+        }
+      } catch {
+      }
+    }
+    results.push(caseObj);
+  }
+  return results;
+}
 async function expandFileReferences(tests, evalFileDir) {
   const expanded = [];
   for (const entry of tests) {
@@ -5585,11 +5642,11 @@ function createFilesystemTools(workspacePath) {
       execute: async (input) => {
         try {
           const resolved = resolveSandboxed(workspacePath, input.path);
-          const stat12 = await import_promises12.default.stat(resolved);
-          if (stat12.isDirectory()) {
+          const stat14 = await import_promises12.default.stat(resolved);
+          if (stat14.isDirectory()) {
             return { error: `'${input.path}' is a directory, not a file` };
           }
-          const buffer = Buffer.alloc(Math.min(stat12.size, MAX_FILE_SIZE));
+          const buffer = Buffer.alloc(Math.min(stat14.size, MAX_FILE_SIZE));
           const fd = await import_promises12.default.open(resolved, "r");
           try {
             await fd.read(buffer, 0, buffer.length, 0);
@@ -5597,8 +5654,8 @@ function createFilesystemTools(workspacePath) {
             await fd.close();
           }
           const content = buffer.toString("utf-8");
-          const truncated = stat12.size > MAX_FILE_SIZE;
-          return { content, truncated, size: stat12.size };
+          const truncated = stat14.size > MAX_FILE_SIZE;
+          return { content, truncated, size: stat14.size };
         } catch (error) {
           return { error: error instanceof Error ? error.message : String(error) };
         }
@@ -5649,8 +5706,8 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
       const ext = import_node_path12.default.extname(entry.name).toLowerCase();
       if (BINARY_EXTENSIONS.has(ext)) continue;
       try {
-        const stat12 = await import_promises12.default.stat(fullPath);
-        if (stat12.size > MAX_FILE_SIZE) continue;
+        const stat14 = await import_promises12.default.stat(fullPath);
+        if (stat14.size > MAX_FILE_SIZE) continue;
         const content = await import_promises12.default.readFile(fullPath, "utf-8");
         const lines = content.split("\n");
         for (let i = 0; i < lines.length; i++) {
@@ -22501,7 +22558,17 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
   let expandedTestCases;
   if (typeof rawTestCases === "string") {
     const externalPath = import_node_path50.default.resolve(evalFileDir, rawTestCases);
-    expandedTestCases = await loadCasesFromFile(externalPath);
+    let isDir = false;
+    try {
+      const pathStat = await (0, import_promises36.stat)(externalPath);
+      isDir = pathStat.isDirectory();
+    } catch {
+    }
+    if (isDir) {
+      expandedTestCases = await loadCasesFromDirectory(externalPath);
+    } else {
+      expandedTestCases = await loadCasesFromFile(externalPath);
+    }
   } else if (Array.isArray(rawTestCases)) {
     expandedTestCases = await expandFileReferences(rawTestCases, evalFileDir);
   } else {
@@ -25313,32 +25380,44 @@ var import_node_path55 = __toESM(require("path"), 1);
 var import_yaml10 = require("yaml");
 init_paths();
 function getBenchmarksRegistryPath() {
-  return import_node_path55.default.join(getAgentvConfigDir(), "projects.yaml");
+  return import_node_path55.default.join(getAgentvConfigDir(), "benchmarks.yaml");
 }
-function migrateProjectsYaml(targetPath) {
-  const dataHome = getAgentvHome();
-  const configDir = getAgentvConfigDir();
-  if (dataHome === configDir) return;
-  const legacyPath = import_node_path55.default.join(dataHome, "projects.yaml");
-  if (!(0, import_node_fs20.existsSync)(legacyPath)) return;
-  (0, import_node_fs20.mkdirSync)(import_node_path55.default.dirname(targetPath), { recursive: true });
-  (0, import_node_fs20.copyFileSync)(legacyPath, targetPath);
+function fromYaml(raw) {
+  if (!raw || typeof raw !== "object") return null;
+  const e = raw;
+  if (typeof e.id !== "string" || typeof e.name !== "string" || typeof e.path !== "string") {
+    return null;
+  }
+  return {
+    id: e.id,
+    name: e.name,
+    path: e.path,
+    addedAt: typeof e.added_at === "string" ? e.added_at : "",
+    lastOpenedAt: typeof e.last_opened_at === "string" ? e.last_opened_at : ""
+  };
+}
+function toYaml(entry) {
+  return {
+    id: entry.id,
+    name: entry.name,
+    path: entry.path,
+    added_at: entry.addedAt,
+    last_opened_at: entry.lastOpenedAt
+  };
 }
 function loadBenchmarkRegistry() {
   const registryPath = getBenchmarksRegistryPath();
-  if (!(0, import_node_fs20.existsSync)(registryPath)) {
-    migrateProjectsYaml(registryPath);
-  }
   if (!(0, import_node_fs20.existsSync)(registryPath)) {
     return { benchmarks: [] };
   }
   try {
     const raw = (0, import_node_fs20.readFileSync)(registryPath, "utf-8");
     const parsed = (0, import_yaml10.parse)(raw);
-    if (!parsed || !Array.isArray(parsed.benchmarks)) {
+    if (!parsed || typeof parsed !== "object") {
       return { benchmarks: [] };
     }
-    return { benchmarks: parsed.benchmarks };
+    const benchmarks = Array.isArray(parsed.benchmarks) ? parsed.benchmarks.map(fromYaml).filter((e) => e !== null) : [];
+    return { benchmarks };
   } catch {
     return { benchmarks: [] };
   }
@@ -25349,7 +25428,8 @@ function saveBenchmarkRegistry(registry) {
   if (!(0, import_node_fs20.existsSync)(dir)) {
     (0, import_node_fs20.mkdirSync)(dir, { recursive: true });
   }
-  (0, import_node_fs20.writeFileSync)(registryPath, (0, import_yaml10.stringify)({ benchmarks: registry.benchmarks }), "utf-8");
+  const payload = { benchmarks: registry.benchmarks.map(toYaml) };
+  (0, import_node_fs20.writeFileSync)(registryPath, (0, import_yaml10.stringify)(payload), "utf-8");
 }
 function deriveBenchmarkId(dirPath, existingIds) {
   const base = import_node_path55.default.basename(dirPath).toLowerCase().replace(/[^a-z0-9-]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
@@ -25432,7 +25512,7 @@ function discoverBenchmarks(rootDir, maxDepth = 2) {
     }
   }
   scan(absRoot, 0);
-  return results;
+  return results.sort();
 }
 
 // src/evaluation/baseline.ts