@agentv/core 4.32.0-next.1 → 4.34.0-next.1

package/dist/evaluation/validation/index.d.cts

@@ -61,7 +61,9 @@ declare function validateTargetsFile(filePath: string): Promise<ValidationResult
 /**
  * Validate a config.yaml file for schema compliance and structural correctness.
  */
-declare function validateConfigFile(filePath: string): Promise<ValidationResult>;
+declare function validateConfigFile(filePath: string, options?: {
+    scope?: 'project' | 'global';
+}): Promise<ValidationResult>;
 
 /**
  * Validate that file references in eval file content exist.

package/dist/evaluation/validation/index.d.ts

@@ -61,7 +61,9 @@ declare function validateTargetsFile(filePath: string): Promise<ValidationResult
 /**
  * Validate a config.yaml file for schema compliance and structural correctness.
  */
-declare function validateConfigFile(filePath: string): Promise<ValidationResult>;
+declare function validateConfigFile(filePath: string, options?: {
+    scope?: 'project' | 'global';
+}): Promise<ValidationResult>;
 
 /**
  * Validate that file references in eval file content exist.

package/dist/evaluation/validation/index.js

@@ -6,13 +6,14 @@ import {
   buildSearchRoots,
   findDeprecatedCamelCaseTargetWarnings,
   findGitRoot,
+  getAgentvConfigDir,
   interpolateEnv,
   isGraderKind,
   loadCasesFromDirectory,
   loadCasesFromFile,
   parseYamlValue,
   resolveFileReference
-} from "../../chunk-5RQMJZDJ.js";
+} from "../../chunk-EW5X2RGJ.js";
 
 // src/evaluation/validation/file-type.ts
 import { readFile } from "node:fs/promises";
@@ -947,6 +948,9 @@ import path4 from "node:path";
 function isObject3(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
+function isNonEmptyString(value) {
+  return typeof value === "string" && value.trim().length > 0;
+}
 var COMMON_SETTINGS = new Set(COMMON_TARGET_SETTINGS);
 var RETRY_SETTINGS = /* @__PURE__ */ new Set([
   "max_retries",
@@ -1015,6 +1019,7 @@ var GEMINI_SETTINGS = /* @__PURE__ */ new Set([
 var CODEX_SETTINGS = /* @__PURE__ */ new Set([
   ...COMMON_SETTINGS,
   "model",
+  "model_reasoning_effort",
   "executable",
   "command",
   "binary",
@@ -1024,8 +1029,7 @@ var CODEX_SETTINGS = /* @__PURE__ */ new Set([
   "timeout_seconds",
   "log_dir",
   "log_directory",
-  "log_format",
-  "log_output_format",
+  "stream_log",
   "system_prompt"
 ]);
 var COPILOT_SDK_SETTINGS = /* @__PURE__ */ new Set([
@@ -1150,6 +1154,26 @@ function validateUnknownSettings(target, provider, absolutePath, location, error
         "api_format",
         "The 'api_format' field is no longer supported on Azure targets. AgentV always uses Azure's Responses API (`/openai/v1/responses`). If your deployment only exposes /chat/completions, use 'provider: openai' with a deployment-scoped 'base_url' instead."
       ]
+    ]),
+    codex: /* @__PURE__ */ new Map([
+      [
+        "log_format",
+        "The 'log_format' field is no longer supported on Codex targets. Use 'stream_log: raw' for per-event logs or 'stream_log: summary' for consolidated logs."
+      ],
+      [
+        "log_output_format",
+        "The 'log_output_format' field is no longer supported on Codex targets. Use 'stream_log: raw' for per-event logs or 'stream_log: summary' for consolidated logs."
+      ]
+    ]),
+    "codex-cli": /* @__PURE__ */ new Map([
+      [
+        "log_format",
+        "The 'log_format' field is no longer supported on Codex targets. Use 'stream_log: raw' for per-event logs or 'stream_log: summary' for consolidated logs."
+      ],
+      [
+        "log_output_format",
+        "The 'log_output_format' field is no longer supported on Codex targets. Use 'stream_log: raw' for per-event logs or 'stream_log: summary' for consolidated logs."
+      ]
     ])
   };
   const removedForProvider = removedPerProvider[provider];
@@ -1186,9 +1210,11 @@ async function validateTargetsFile(filePath) {
   const errors = [];
   const absolutePath = path4.resolve(filePath);
   let parsed;
+  let rawParsed;
   try {
     const content = await readFile4(absolutePath, "utf8");
-    parsed = interpolateEnv(parseYamlValue(content), process.env);
+    rawParsed = parseYamlValue(content);
+    parsed = interpolateEnv(rawParsed, process.env);
   } catch (error) {
     errors.push({
       severity: "error",
@@ -1309,6 +1335,7 @@ async function validateTargetsFile(filePath) {
     };
   }
   const targets = parsed.targets;
+  const rawTargets = isObject3(rawParsed) && Array.isArray(rawParsed.targets) ? rawParsed.targets : [];
   if (!Array.isArray(targets)) {
     errors.push({
       severity: "error",
@@ -1358,7 +1385,9 @@ async function validateTargetsFile(filePath) {
       });
     }
     const provider = target.provider;
-    const hasUseTarget = typeof target.use_target === "string" && target.use_target.trim().length > 0;
+    const rawTarget = rawTargets[i];
+    const rawUseTarget = isObject3(rawTarget) ? rawTarget.use_target : void 0;
+    const hasUseTarget = isNonEmptyString(target.use_target) || isNonEmptyString(rawUseTarget);
     const providerValue = typeof provider === "string" ? provider.trim().toLowerCase() : void 0;
     const isTemplated = typeof provider === "string" && /^\$\{\{.+\}\}$/.test(provider.trim());
     if (!hasUseTarget && (typeof provider !== "string" || provider.trim().length === 0)) {
@@ -1402,8 +1431,10 @@ async function validateTargetsFile(filePath) {
 
 // src/evaluation/validation/config-validator.ts
 import { readFile as readFile5 } from "node:fs/promises";
-async function validateConfigFile(filePath) {
+import path5 from "node:path";
+async function validateConfigFile(filePath, options = {}) {
   const errors = [];
+  const scope = options.scope ?? inferConfigScope(filePath);
   try {
     const content = await readFile5(filePath, "utf8");
     const parsed = interpolateEnv(parseYamlValue(content), process.env);
@@ -1452,78 +1483,43 @@ async function validateConfigFile(filePath) {
         });
       }
     }
-    const results = config.results;
-    if (results !== void 0) {
-      if (typeof results !== "object" || results === null || Array.isArray(results)) {
+    validateResultsConfig(errors, filePath, config.results, "results");
+    const projects = config.projects;
+    if (projects !== void 0) {
+      if (scope === "project") {
+        errors.push({
+          severity: "warning",
+          filePath,
+          location: "projects",
+          message: "Field 'projects' is only valid in $AGENTV_HOME/config.yaml. Ignoring project registry entries in project-local .agentv/config.yaml."
+        });
+      } else if (!Array.isArray(projects)) {
         errors.push({
           severity: "error",
           filePath,
-          location: "results",
-          message: "Field 'results' must be an object"
+          location: "projects",
+          message: "Field 'projects' must be an array"
         });
       } else {
-        const resultsRecord = results;
-        if (resultsRecord.mode !== "github") {
-          errors.push({
-            severity: "error",
-            filePath,
-            location: "results.mode",
-            message: "Field 'results.mode' must be 'github'"
-          });
-        }
-        if (typeof resultsRecord.repo !== "string" || resultsRecord.repo.trim().length === 0) {
-          errors.push({
-            severity: "error",
-            filePath,
-            location: "results.repo",
-            message: "Field 'results.repo' must be a non-empty string"
-          });
-        }
-        if (resultsRecord.path !== void 0) {
-          if (typeof resultsRecord.path !== "string" || resultsRecord.path.trim().length === 0) {
-            errors.push({
-              severity: "error",
-              filePath,
-              location: "results.path",
-              message: "Field 'results.path' must be a non-empty string"
-            });
-          } else {
-            const p = resultsRecord.path.trim();
-            const isFilesystemPath = p.startsWith("/") || p.startsWith("~/") || p.startsWith("~\\") || p === "~" || /^[A-Za-z]:[/\\]/.test(p);
-            if (!isFilesystemPath) {
-              errors.push({
-                severity: "error",
-                filePath,
-                location: "results.path",
-                message: `'results.path' must be an absolute or home-relative filesystem path (e.g., ~/data/agentv-results). Found: '${p}'. Remove 'path' to use the default.`
-              });
-            }
-          }
-        }
-        if (resultsRecord.auto_push !== void 0 && typeof resultsRecord.auto_push !== "boolean") {
-          errors.push({
-            severity: "error",
-            filePath,
-            location: "results.auto_push",
-            message: "Field 'results.auto_push' must be a boolean"
-          });
-        }
-        if (resultsRecord.branch_prefix !== void 0 && (typeof resultsRecord.branch_prefix !== "string" || resultsRecord.branch_prefix.trim().length === 0)) {
-          errors.push({
-            severity: "error",
-            filePath,
-            location: "results.branch_prefix",
-            message: "Field 'results.branch_prefix' must be a non-empty string"
-          });
-        }
+        validateProjects(errors, filePath, projects);
       }
     }
+    if (config.results_by_project !== void 0) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location: "results_by_project",
+        message: "Field 'results_by_project' is deprecated. Put per-project result repo settings under projects[].results in $AGENTV_HOME/config.yaml."
+      });
+    }
     const allowedFields = /* @__PURE__ */ new Set([
       "$schema",
       "eval_patterns",
       "required_version",
       "execution",
       "results",
+      "projects",
+      "results_by_project",
       "dashboard",
       "studio"
     ]);
@@ -1550,16 +1546,115 @@ async function validateConfigFile(filePath) {
     return { valid: false, filePath, fileType: "config", errors };
   }
 }
+function inferConfigScope(filePath) {
+  const globalConfigPath = path5.resolve(getAgentvConfigDir(), "config.yaml");
+  if (path5.resolve(filePath) === globalConfigPath) {
+    return "global";
+  }
+  return filePath.split(/[\\/]/).includes(".agentv") ? "project" : "global";
+}
+function validateProjects(errors, filePath, projects) {
+  projects.forEach((project, index) => {
+    const location = `projects[${index}]`;
+    if (typeof project !== "object" || project === null || Array.isArray(project)) {
+      errors.push({
+        severity: "error",
+        filePath,
+        location,
+        message: `Field '${location}' must be an object`
+      });
+      return;
+    }
+    const projectRecord = project;
+    validateRequiredString(errors, filePath, projectRecord.id, `${location}.id`);
+    validateRequiredString(errors, filePath, projectRecord.name, `${location}.name`);
+    validateRequiredString(errors, filePath, projectRecord.path, `${location}.path`);
+    validateResultsConfig(errors, filePath, projectRecord.results, `${location}.results`);
+  });
+}
+function validateRequiredString(errors, filePath, value, location) {
+  if (typeof value !== "string" || value.trim().length === 0) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location,
+      message: `Field '${location}' must be a non-empty string`
+    });
+  }
+}
+function validateResultsConfig(errors, filePath, rawResults, location) {
+  if (rawResults === void 0) {
+    return;
+  }
+  if (typeof rawResults !== "object" || rawResults === null || Array.isArray(rawResults)) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location,
+      message: `Field '${location}' must be an object`
+    });
+    return;
+  }
+  const resultsRecord = rawResults;
+  if (resultsRecord.mode !== "github") {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.mode`,
+      message: `Field '${location}.mode' must be 'github'`
+    });
+  }
+  validateRequiredString(errors, filePath, resultsRecord.repo, `${location}.repo`);
+  if (resultsRecord.path !== void 0) {
+    if (typeof resultsRecord.path !== "string" || resultsRecord.path.trim().length === 0) {
+      errors.push({
+        severity: "error",
+        filePath,
+        location: `${location}.path`,
+        message: `Field '${location}.path' must be a non-empty string`
+      });
+    } else {
+      const p = resultsRecord.path.trim();
+      if (!isFilesystemPath(p)) {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: `${location}.path`,
+          message: `'${location}.path' must be an absolute or home-relative filesystem path (e.g., ~/data/agentv-results). Found: '${p}'. Remove 'path' to use the default.`
+        });
+      }
+    }
+  }
+  if (resultsRecord.auto_push !== void 0 && typeof resultsRecord.auto_push !== "boolean") {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.auto_push`,
+      message: `Field '${location}.auto_push' must be a boolean`
+    });
+  }
+  if (resultsRecord.branch_prefix !== void 0 && (typeof resultsRecord.branch_prefix !== "string" || resultsRecord.branch_prefix.trim().length === 0)) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.branch_prefix`,
+      message: `Field '${location}.branch_prefix' must be a non-empty string`
+    });
+  }
+}
+function isFilesystemPath(p) {
+  return p.startsWith("/") || p.startsWith("~/") || p.startsWith("~\\") || p === "~" || /^[A-Za-z]:[/\\]/.test(p);
+}
 
 // src/evaluation/validation/file-reference-validator.ts
 import { readFile as readFile6 } from "node:fs/promises";
-import path5 from "node:path";
+import path6 from "node:path";
 function isObject4(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 async function validateFileReferences(evalFilePath) {
   const errors = [];
-  const absolutePath = path5.resolve(evalFilePath);
+  const absolutePath = path6.resolve(evalFilePath);
   const gitRoot = await findGitRoot(absolutePath);
   if (!gitRoot) {
     errors.push({
@@ -1684,14 +1779,14 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
 
 // src/evaluation/validation/workspace-path-validator.ts
 import { access, readFile as readFile7 } from "node:fs/promises";
-import path6 from "node:path";
+import path7 from "node:path";
 function isObject5(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 async function validateWorkspacePaths(evalFilePath) {
   const errors = [];
-  const absolutePath = path6.resolve(evalFilePath);
-  const evalDir = path6.dirname(absolutePath);
+  const absolutePath = path7.resolve(evalFilePath);
+  const evalDir = path7.dirname(absolutePath);
   let parsed;
   try {
     const content = await readFile7(absolutePath, "utf8");
@@ -1703,12 +1798,12 @@ async function validateWorkspacePaths(evalFilePath) {
   const workspaceRaw = parsed.workspace;
   if (workspaceRaw === void 0 || workspaceRaw === null) return errors;
   if (typeof workspaceRaw === "string") {
-    const workspaceFilePath = path6.resolve(evalDir, workspaceRaw);
+    const workspaceFilePath = path7.resolve(evalDir, workspaceRaw);
     try {
       const wsContent = await readFile7(workspaceFilePath, "utf8");
       const wsParsed = parseYamlValue(wsContent);
       if (isObject5(wsParsed)) {
-        const wsDir = path6.dirname(workspaceFilePath);
+        const wsDir = path7.dirname(workspaceFilePath);
         await validateWorkspaceObject(wsParsed, wsDir, absolutePath, "workspace", errors);
       }
     } catch {
@@ -1721,7 +1816,7 @@ async function validateWorkspacePaths(evalFilePath) {
 async function validateWorkspaceObject(obj, baseDir, evalFilePath, location, errors) {
   const template = obj.template;
   if (typeof template === "string") {
-    const templatePath = path6.isAbsolute(template) ? template : path6.resolve(baseDir, template);
+    const templatePath = path7.isAbsolute(template) ? template : path7.resolve(baseDir, template);
     if (!await fileExists(templatePath)) {
       errors.push({
         severity: "error",
@@ -1737,14 +1832,14 @@ async function validateWorkspaceObject(obj, baseDir, evalFilePath, location, err
     const hook = hooks[hookName];
     if (!isObject5(hook)) continue;
     const hookCwdRaw = typeof hook.cwd === "string" ? hook.cwd : void 0;
-    const hookCwd = hookCwdRaw ? path6.isAbsolute(hookCwdRaw) ? hookCwdRaw : path6.resolve(baseDir, hookCwdRaw) : baseDir;
+    const hookCwd = hookCwdRaw ? path7.isAbsolute(hookCwdRaw) ? hookCwdRaw : path7.resolve(baseDir, hookCwdRaw) : baseDir;
     const command = hook.command ?? hook.script;
     if (!Array.isArray(command)) continue;
     for (let i = 0; i < command.length; i++) {
       const arg = command[i];
       if (typeof arg !== "string") continue;
       if (!looksLikeFilePath(arg)) continue;
-      const resolved = path6.isAbsolute(arg) ? arg : path6.resolve(hookCwd, arg);
+      const resolved = path7.isAbsolute(arg) ? arg : path7.resolve(hookCwd, arg);
       if (!await fileExists(resolved)) {
         errors.push({
           severity: "error",