@agentv/core 4.28.0 → 4.29.1-next.1

package/dist/index.cjs

@@ -24667,7 +24667,7 @@ __export(index_exports, {
   TranscriptProvider: () => TranscriptProvider,
   WorkspaceCreationError: () => WorkspaceCreationError,
   WorkspacePoolManager: () => WorkspacePoolManager,
-  addBenchmark: () => addBenchmark,
+  addProject: () => addProject,
   assembleLlmGraderPrompt: () => assembleLlmGraderPrompt,
   avgToolDurationMs: () => avgToolDurationMs,
   buildDirectoryChain: () => buildDirectoryChain2,
@@ -24698,17 +24698,17 @@ __export(index_exports, {
   createTempWorkspace: () => createTempWorkspace,
   deepEqual: () => deepEqual,
   defineConfig: () => defineConfig,
-  deriveBenchmarkId: () => deriveBenchmarkId,
   deriveCategory: () => deriveCategory,
+  deriveProjectId: () => deriveProjectId,
   detectFormat: () => detectFormat,
   directPushResults: () => directPushResults,
   directorySizeBytes: () => directorySizeBytes,
   discoverAssertions: () => discoverAssertions,
-  discoverBenchmarks: () => discoverBenchmarks,
   discoverClaudeSessions: () => discoverClaudeSessions,
   discoverCodexSessions: () => discoverCodexSessions,
   discoverCopilotSessions: () => discoverCopilotSessions,
   discoverGraders: () => discoverGraders,
+  discoverProjects: () => discoverProjects,
   discoverProviders: () => discoverProviders,
   ensureResultsRepoClone: () => ensureResultsRepoClone,
   ensureVSCodeSubagents: () => ensureVSCodeSubagents,
@@ -24735,9 +24735,9 @@ __export(index_exports, {
   generateRubrics: () => generateRubrics,
   getAgentvConfigDir: () => getAgentvConfigDir,
   getAgentvHome: () => getAgentvHome,
-  getBenchmark: () => getBenchmark,
-  getBenchmarksRegistryPath: () => getBenchmarksRegistryPath,
   getOutputFilenames: () => getOutputFilenames,
+  getProject: () => getProject,
+  getProjectsRegistryPath: () => getProjectsRegistryPath,
   getResultsRepoCachePaths: () => getResultsRepoCachePaths,
   getResultsRepoStatus: () => getResultsRepoStatus,
   getSubagentsRoot: () => getSubagentsRoot,
@@ -24759,11 +24759,11 @@ __export(index_exports, {
   isTestMessageRole: () => isTestMessageRole,
   killAllTrackedChildren: () => killAllTrackedChildren,
   listTargetNames: () => listTargetNames,
-  loadBenchmarkRegistry: () => loadBenchmarkRegistry,
   loadConfig: () => loadConfig,
   loadEvalCaseById: () => loadEvalCaseById,
   loadEvalCases: () => loadEvalCases,
   loadEvalSuite: () => loadEvalSuite,
+  loadProjectRegistry: () => loadProjectRegistry,
   loadTestById: () => loadTestById,
   loadTestSuite: () => loadTestSuite,
   loadTests: () => loadTests,
@@ -24789,7 +24789,7 @@ __export(index_exports, {
   readTextFile: () => readTextFile,
   readTranscriptFile: () => readTranscriptFile,
   readTranscriptJsonl: () => readTranscriptJsonl,
-  removeBenchmark: () => removeBenchmark,
+  removeProject: () => removeProject,
   resolveAndCreateProvider: () => resolveAndCreateProvider,
   resolveDelegatedTargetDefinition: () => resolveDelegatedTargetDefinition,
   resolveFileReference: () => resolveFileReference3,
@@ -24812,7 +24812,7 @@ __export(index_exports, {
   runIsJsonAssertion: () => runIsJsonAssertion,
   runRegexAssertion: () => runRegexAssertion,
   runStartsWithAssertion: () => runStartsWithAssertion,
-  saveBenchmarkRegistry: () => saveBenchmarkRegistry,
+  saveProjectRegistry: () => saveProjectRegistry,
   scanRepoDeps: () => scanRepoDeps,
   scoreRangeEvaluationSchema: () => scoreRangeEvaluationSchema,
   scoreToVerdict: () => scoreToVerdict,
@@ -24825,14 +24825,14 @@ __export(index_exports, {
   subscribeToCopilotSdkLogEntries: () => subscribeToCopilotSdkLogEntries,
   subscribeToPiLogEntries: () => subscribeToPiLogEntries,
   substituteVariables: () => substituteVariables,
-  syncBenchmark: () => syncBenchmark,
-  syncBenchmarks: () => syncBenchmarks,
+  syncProject: () => syncProject,
+  syncProjects: () => syncProjects,
   syncResultsRepo: () => syncResultsRepo,
   toCamelCaseDeep: () => toCamelCaseDeep,
   toSnakeCaseDeep: () => toSnakeCaseDeep,
   toTranscriptJsonLines: () => toTranscriptJsonLines,
   tokensPerTool: () => tokensPerTool,
-  touchBenchmark: () => touchBenchmark,
+  touchProject: () => touchProject,
   trackChild: () => trackChild,
   trackedChildCount: () => trackedChildCount,
   transpileEvalYaml: () => transpileEvalYaml,
@@ -25766,7 +25766,7 @@ async function directPushResults(params) {
 // src/index.ts
 init_paths();
 
-// src/benchmarks.ts
+// src/projects.ts
 init_cjs_shims();
 var import_node_fs20 = require("fs");
 var import_node_path55 = __toESM(require("path"), 1);
@@ -25774,9 +25774,56 @@ var import_yaml2 = require("yaml");
 init_interpolation();
 init_yaml_loader();
 init_paths();
-function getBenchmarksRegistryPath() {
+function getProjectsRegistryPath() {
+  return import_node_path55.default.join(getAgentvConfigDir(), "projects.yaml");
+}
+function getLegacyBenchmarksRegistryPath() {
   return import_node_path55.default.join(getAgentvConfigDir(), "benchmarks.yaml");
 }
+function migrateLegacyBenchmarksFile() {
+  const newPath = getProjectsRegistryPath();
+  const oldPath = getLegacyBenchmarksRegistryPath();
+  const newExists = (0, import_node_fs20.existsSync)(newPath);
+  const oldExists = (0, import_node_fs20.existsSync)(oldPath);
+  if (!oldExists) return;
+  if (newExists) {
+    console.warn(
+      `[agentv] Both ${oldPath} and ${newPath} exist. Using ${import_node_path55.default.basename(newPath)}; delete ${import_node_path55.default.basename(oldPath)} when you've confirmed the new file is correct.`
+    );
+    return;
+  }
+  let parsed = null;
+  try {
+    const raw = (0, import_node_fs20.readFileSync)(oldPath, "utf-8");
+    parsed = parseYamlValue(raw);
+  } catch (err) {
+    console.warn(
+      `[agentv] Failed to read legacy ${import_node_path55.default.basename(oldPath)} for migration: ${err.message}. Leaving the file in place; you may need to migrate it manually.`
+    );
+    return;
+  }
+  const entries = parsed && typeof parsed === "object" && Array.isArray(parsed.benchmarks) ? parsed.benchmarks : [];
+  const newContent = (0, import_yaml2.stringify)({ projects: entries });
+  const tempPath = `${newPath}.migrating`;
+  try {
+    (0, import_node_fs20.mkdirSync)(import_node_path55.default.dirname(newPath), { recursive: true });
+    (0, import_node_fs20.writeFileSync)(tempPath, newContent, "utf-8");
+    (0, import_node_fs20.renameSync)(tempPath, newPath);
+    (0, import_node_fs20.unlinkSync)(oldPath);
+  } catch (err) {
+    try {
+      if ((0, import_node_fs20.existsSync)(tempPath)) (0, import_node_fs20.unlinkSync)(tempPath);
+    } catch {
+    }
+    console.warn(
+      `[agentv] Failed to migrate ${import_node_path55.default.basename(oldPath)} \u2192 ${import_node_path55.default.basename(newPath)}: ${err.message}. Legacy file left in place.`
+    );
+    return;
+  }
+  console.log(
+    `[agentv] Migrated registry: ${import_node_path55.default.basename(oldPath)} \u2192 ${import_node_path55.default.basename(newPath)} (${entries.length} entr${entries.length === 1 ? "y" : "ies"})`
+  );
+}
 function fromYaml(raw) {
   if (!raw || typeof raw !== "object") return null;
   const e = raw;
@@ -25811,36 +25858,37 @@ function toYaml(entry) {
   }
   return yaml;
 }
-function loadBenchmarkRegistry() {
-  const registryPath = getBenchmarksRegistryPath();
+function loadProjectRegistry() {
+  migrateLegacyBenchmarksFile();
+  const registryPath = getProjectsRegistryPath();
   if (!(0, import_node_fs20.existsSync)(registryPath)) {
-    return { benchmarks: [] };
+    return { projects: [] };
   }
   try {
     const raw = (0, import_node_fs20.readFileSync)(registryPath, "utf-8");
     const parsed = parseYamlValue(raw);
     if (!parsed || typeof parsed !== "object") {
-      return { benchmarks: [] };
+      return { projects: [] };
     }
     const env = process.env;
-    const benchmarks = Array.isArray(parsed.benchmarks) ? parsed.benchmarks.map((e) => fromYaml(interpolateEnv(e, env))).filter((e) => e !== null) : [];
-    return { benchmarks };
+    const projects = Array.isArray(parsed.projects) ? parsed.projects.map((e) => fromYaml(interpolateEnv(e, env))).filter((e) => e !== null) : [];
+    return { projects };
   } catch {
-    return { benchmarks: [] };
+    return { projects: [] };
   }
 }
-function saveBenchmarkRegistry(registry) {
-  const registryPath = getBenchmarksRegistryPath();
+function saveProjectRegistry(registry) {
+  const registryPath = getProjectsRegistryPath();
   const dir = import_node_path55.default.dirname(registryPath);
   if (!(0, import_node_fs20.existsSync)(dir)) {
     (0, import_node_fs20.mkdirSync)(dir, { recursive: true });
   }
-  const payload = { benchmarks: registry.benchmarks.map(toYaml) };
+  const payload = { projects: registry.projects.map(toYaml) };
   (0, import_node_fs20.writeFileSync)(registryPath, (0, import_yaml2.stringify)(payload), "utf-8");
 }
-function deriveBenchmarkId(dirPath, existingIds) {
+function deriveProjectId(dirPath, existingIds) {
   const base = import_node_path55.default.basename(dirPath).toLowerCase().replace(/[^a-z0-9-]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
-  let candidate = base || "benchmark";
+  let candidate = base || "project";
   let suffix = 2;
   while (existingIds.includes(candidate)) {
     candidate = `${base}-${suffix}`;
@@ -25848,54 +25896,54 @@ function deriveBenchmarkId(dirPath, existingIds) {
   }
   return candidate;
 }
-function addBenchmark(benchmarkPath) {
-  const absPath = import_node_path55.default.resolve(benchmarkPath);
+function addProject(projectPath) {
+  const absPath = import_node_path55.default.resolve(projectPath);
   if (!(0, import_node_fs20.existsSync)(absPath)) {
     throw new Error(`Directory not found: ${absPath}`);
   }
   if (!(0, import_node_fs20.existsSync)(import_node_path55.default.join(absPath, ".agentv"))) {
     throw new Error(`No .agentv/ directory found in ${absPath}. Run an evaluation first.`);
   }
-  const registry = loadBenchmarkRegistry();
-  const existing = registry.benchmarks.find((p) => p.path === absPath);
+  const registry = loadProjectRegistry();
+  const existing = registry.projects.find((p) => p.path === absPath);
   if (existing) {
     return existing;
   }
   const now = (/* @__PURE__ */ new Date()).toISOString();
   const entry = {
-    id: deriveBenchmarkId(
+    id: deriveProjectId(
       absPath,
-      registry.benchmarks.map((p) => p.id)
+      registry.projects.map((p) => p.id)
     ),
     name: import_node_path55.default.basename(absPath),
     path: absPath,
     addedAt: now,
     lastOpenedAt: now
   };
-  registry.benchmarks.push(entry);
-  saveBenchmarkRegistry(registry);
+  registry.projects.push(entry);
+  saveProjectRegistry(registry);
   return entry;
 }
-function removeBenchmark(benchmarkId) {
-  const registry = loadBenchmarkRegistry();
-  const idx = registry.benchmarks.findIndex((p) => p.id === benchmarkId);
+function removeProject(projectId) {
+  const registry = loadProjectRegistry();
+  const idx = registry.projects.findIndex((p) => p.id === projectId);
   if (idx < 0) return false;
-  registry.benchmarks.splice(idx, 1);
-  saveBenchmarkRegistry(registry);
+  registry.projects.splice(idx, 1);
+  saveProjectRegistry(registry);
   return true;
 }
-function getBenchmark(benchmarkId) {
-  return loadBenchmarkRegistry().benchmarks.find((p) => p.id === benchmarkId);
+function getProject(projectId) {
+  return loadProjectRegistry().projects.find((p) => p.id === projectId);
 }
-function touchBenchmark(benchmarkId) {
-  const registry = loadBenchmarkRegistry();
-  const entry = registry.benchmarks.find((p) => p.id === benchmarkId);
+function touchProject(projectId) {
+  const registry = loadProjectRegistry();
+  const entry = registry.projects.find((p) => p.id === projectId);
   if (entry) {
     entry.lastOpenedAt = (/* @__PURE__ */ new Date()).toISOString();
-    saveBenchmarkRegistry(registry);
+    saveProjectRegistry(registry);
   }
 }
-function discoverBenchmarks(rootDir, maxDepth = 2) {
+function discoverProjects(rootDir, maxDepth = 2) {
   const absRoot = import_node_path55.default.resolve(rootDir);
   if (!(0, import_node_fs20.existsSync)(absRoot) || !(0, import_node_fs20.statSync)(absRoot).isDirectory()) {
     return [];
@@ -25922,13 +25970,13 @@ function discoverBenchmarks(rootDir, maxDepth = 2) {
   return results.sort();
 }
 
-// src/benchmark-sync.ts
+// src/project-sync.ts
 init_cjs_shims();
 var childProcess = __toESM(require("child_process"), 1);
 var import_node_fs21 = require("fs");
-async function syncBenchmark(entry) {
+async function syncProject(entry) {
   if (!entry.source) {
-    throw new Error(`Benchmark '${entry.id}' has no source defined`);
+    throw new Error(`Project '${entry.id}' has no source defined`);
   }
   const { url, ref } = entry.source;
   const dest = entry.path;
@@ -25942,12 +25990,12 @@ async function syncBenchmark(entry) {
     );
   }
 }
-async function syncBenchmarks(entries) {
+async function syncProjects(entries) {
   for (const entry of entries) {
     if (!entry.source) continue;
-    console.log(`Syncing benchmark '${entry.id}' from ${entry.source.url}...`);
-    await syncBenchmark(entry);
-    console.log(`Benchmark '${entry.id}' synced.`);
+    console.log(`Syncing project '${entry.id}' from ${entry.source.url}...`);
+    await syncProject(entry);
+    console.log(`Project '${entry.id}' synced.`);
   }
 }
 
@@ -27282,7 +27330,7 @@ function createAgentKernel() {
   TranscriptProvider,
   WorkspaceCreationError,
   WorkspacePoolManager,
-  addBenchmark,
+  addProject,
   assembleLlmGraderPrompt,
   avgToolDurationMs,
   buildDirectoryChain,
@@ -27313,17 +27361,17 @@ function createAgentKernel() {
   createTempWorkspace,
   deepEqual,
   defineConfig,
-  deriveBenchmarkId,
   deriveCategory,
+  deriveProjectId,
   detectFormat,
   directPushResults,
   directorySizeBytes,
   discoverAssertions,
-  discoverBenchmarks,
   discoverClaudeSessions,
   discoverCodexSessions,
   discoverCopilotSessions,
   discoverGraders,
+  discoverProjects,
   discoverProviders,
   ensureResultsRepoClone,
   ensureVSCodeSubagents,
@@ -27350,9 +27398,9 @@ function createAgentKernel() {
   generateRubrics,
   getAgentvConfigDir,
   getAgentvHome,
-  getBenchmark,
-  getBenchmarksRegistryPath,
   getOutputFilenames,
+  getProject,
+  getProjectsRegistryPath,
   getResultsRepoCachePaths,
   getResultsRepoStatus,
   getSubagentsRoot,
@@ -27374,11 +27422,11 @@ function createAgentKernel() {
   isTestMessageRole,
   killAllTrackedChildren,
   listTargetNames,
-  loadBenchmarkRegistry,
   loadConfig,
   loadEvalCaseById,
   loadEvalCases,
   loadEvalSuite,
+  loadProjectRegistry,
   loadTestById,
   loadTestSuite,
   loadTests,
@@ -27404,7 +27452,7 @@ function createAgentKernel() {
   readTextFile,
   readTranscriptFile,
   readTranscriptJsonl,
-  removeBenchmark,
+  removeProject,
   resolveAndCreateProvider,
   resolveDelegatedTargetDefinition,
   resolveFileReference,
@@ -27427,7 +27475,7 @@ function createAgentKernel() {
   runIsJsonAssertion,
   runRegexAssertion,
   runStartsWithAssertion,
-  saveBenchmarkRegistry,
+  saveProjectRegistry,
   scanRepoDeps,
   scoreRangeEvaluationSchema,
   scoreToVerdict,
@@ -27440,14 +27488,14 @@ function createAgentKernel() {
   subscribeToCopilotSdkLogEntries,
   subscribeToPiLogEntries,
   substituteVariables,
-  syncBenchmark,
-  syncBenchmarks,
+  syncProject,
+  syncProjects,
   syncResultsRepo,
   toCamelCaseDeep,
   toSnakeCaseDeep,
   toTranscriptJsonLines,
   tokensPerTool,
-  touchBenchmark,
+  touchProject,
   trackChild,
   trackedChildCount,
   transpileEvalYaml,