@agentv/core 3.7.0 → 3.9.0

package/dist/evaluation/validation/index.cjs

@@ -103,13 +103,56 @@ function getExpectedSchema(fileType) {
 }
 
 // src/evaluation/validation/eval-validator.ts
+var import_promises3 = require("fs/promises");
+var import_node_path3 = __toESM(require("path"), 1);
+var import_yaml3 = require("yaml");
+
+// src/evaluation/interpolation.ts
+var ENV_VAR_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
+function interpolateEnv(value, env) {
+  if (typeof value === "string") {
+    return value.replace(ENV_VAR_PATTERN, (_, varName) => env[varName] ?? "");
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => interpolateEnv(item, env));
+  }
+  if (value !== null && typeof value === "object") {
+    const result = {};
+    for (const [key, val] of Object.entries(value)) {
+      result[key] = interpolateEnv(val, env);
+    }
+    return result;
+  }
+  return value;
+}
+
+// src/evaluation/loaders/case-file-loader.ts
 var import_promises2 = require("fs/promises");
 var import_node_path2 = __toESM(require("path"), 1);
+var import_fast_glob = __toESM(require("fast-glob"), 1);
 var import_yaml2 = require("yaml");
 
 // src/evaluation/types.ts
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
 var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
+function isJsonObject(value) {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    return false;
+  }
+  return Object.values(value).every(isJsonValue);
+}
+function isJsonValue(value) {
+  if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+    return true;
+  }
+  if (Array.isArray(value)) {
+    return value.every(isJsonValue);
+  }
+  if (typeof value === "object") {
+    return isJsonObject(value);
+  }
+  return false;
+}
 var EVALUATOR_KIND_VALUES = [
   "code-grader",
   "llm-grader",
@@ -143,6 +186,74 @@ function isEvaluatorKind(value) {
   return typeof value === "string" && EVALUATOR_KIND_SET.has(value);
 }
 
+// src/evaluation/loaders/case-file-loader.ts
+var ANSI_YELLOW = "\x1B[33m";
+var ANSI_RESET = "\x1B[0m";
+function parseYamlCases(content, filePath) {
+  const raw = (0, import_yaml2.parse)(content);
+  const parsed = interpolateEnv(raw, process.env);
+  if (!Array.isArray(parsed)) {
+    throw new Error(
+      `External test file must contain a YAML array, got ${typeof parsed}: ${filePath}`
+    );
+  }
+  const results = [];
+  for (const item of parsed) {
+    if (!isJsonObject(item)) {
+      throw new Error(`External test file contains non-object entry: ${filePath}`);
+    }
+    results.push(item);
+  }
+  return results;
+}
+function parseJsonlCases(content, filePath) {
+  const lines = content.split("\n");
+  const results = [];
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (line === "") continue;
+    try {
+      const raw = JSON.parse(line);
+      const parsed = interpolateEnv(raw, process.env);
+      if (!isJsonObject(parsed)) {
+        throw new Error("Expected JSON object");
+      }
+      results.push(parsed);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(`Malformed JSONL at line ${i + 1}: ${message}
+  File: ${filePath}`);
+    }
+  }
+  return results;
+}
+async function loadCasesFromFile(filePath) {
+  const ext = import_node_path2.default.extname(filePath).toLowerCase();
+  let content;
+  try {
+    content = await (0, import_promises2.readFile)(filePath, "utf8");
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new Error(`Cannot read external test file: ${filePath}
+  ${message}`);
+  }
+  if (content.trim() === "") {
+    console.warn(
+      `${ANSI_YELLOW}Warning: External test file is empty, skipping: ${filePath}${ANSI_RESET}`
+    );
+    return [];
+  }
+  if (ext === ".yaml" || ext === ".yml") {
+    return parseYamlCases(content, filePath);
+  }
+  if (ext === ".jsonl") {
+    return parseJsonlCases(content, filePath);
+  }
+  throw new Error(
+    `Unsupported external test file format '${ext}': ${filePath}. Supported: .yaml, .yml, .jsonl`
+  );
+}
+
 // src/evaluation/validation/eval-validator.ts
 var ASSERTION_TYPES_WITH_STRING_VALUE = /* @__PURE__ */ new Set([
   "contains",
@@ -165,11 +276,11 @@ function isObject(value) {
 }
 async function validateEvalFile(filePath) {
   const errors = [];
-  const absolutePath = import_node_path2.default.resolve(filePath);
+  const absolutePath = import_node_path3.default.resolve(filePath);
   let parsed;
   try {
-    const content = await (0, import_promises2.readFile)(absolutePath, "utf8");
-    parsed = (0, import_yaml2.parse)(content);
+    const content = await (0, import_promises3.readFile)(absolutePath, "utf8");
+    parsed = interpolateEnv((0, import_yaml3.parse)(content), process.env);
   } catch (error) {
     errors.push({
       severity: "error",
@@ -232,6 +343,31 @@ async function validateEvalFile(filePath) {
   }
   if (typeof cases === "string") {
     validateTestsStringPath(cases, absolutePath, errors);
+    await validateWorkspaceConfig(parsed.workspace, absolutePath, errors, "workspace");
+    const ext = import_node_path3.default.extname(cases).toLowerCase();
+    if (VALID_TEST_FILE_EXTENSIONS.has(ext)) {
+      const externalCasesPath = import_node_path3.default.resolve(import_node_path3.default.dirname(absolutePath), cases);
+      try {
+        const externalCases = await loadCasesFromFile(externalCasesPath);
+        for (let i = 0; i < externalCases.length; i++) {
+          const externalCase = externalCases[i];
+          await validateWorkspaceConfig(
+            externalCase.workspace,
+            absolutePath,
+            errors,
+            `tests[${i}].workspace`
+          );
+        }
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        errors.push({
+          severity: "error",
+          filePath: absolutePath,
+          location: "tests",
+          message
+        });
+      }
+    }
     return {
       valid: errors.filter((e) => e.severity === "error").length === 0,
       filePath: absolutePath,
@@ -339,10 +475,14 @@ async function validateEvalFile(filePath) {
     if (assertField !== void 0) {
       validateAssertArray(assertField, location, absolutePath, errors);
     }
+    await validateWorkspaceConfig(
+      evalCase.workspace,
+      absolutePath,
+      errors,
+      `${location}.workspace`
+    );
   }
-  if (isObject(parsed.workspace)) {
-    validateWorkspaceRepoConfig(parsed.workspace, absolutePath, errors);
-  }
+  await validateWorkspaceConfig(parsed.workspace, absolutePath, errors, "workspace");
   return {
     valid: errors.filter((e) => e.severity === "error").length === 0,
     filePath: absolutePath,
@@ -350,6 +490,41 @@ async function validateEvalFile(filePath) {
     errors
   };
 }
+async function validateWorkspaceConfig(workspace, evalFilePath, errors, location) {
+  if (workspace === void 0) {
+    return;
+  }
+  if (isObject(workspace)) {
+    validateWorkspaceRepoConfig(workspace, evalFilePath, errors);
+    return;
+  }
+  if (typeof workspace !== "string") {
+    return;
+  }
+  const workspacePath = import_node_path3.default.resolve(import_node_path3.default.dirname(evalFilePath), workspace);
+  try {
+    const workspaceContent = await (0, import_promises3.readFile)(workspacePath, "utf8");
+    const parsedWorkspace = interpolateEnv((0, import_yaml3.parse)(workspaceContent), process.env);
+    if (!isObject(parsedWorkspace)) {
+      errors.push({
+        severity: "error",
+        filePath: evalFilePath,
+        location,
+        message: `External workspace file must contain a YAML object: ${workspace}`
+      });
+      return;
+    }
+    validateWorkspaceRepoConfig(parsedWorkspace, workspacePath, errors);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    errors.push({
+      severity: "error",
+      filePath: evalFilePath,
+      location,
+      message: `Failed to load external workspace file '${workspace}': ${message}`
+    });
+  }
+}
 function validateWorkspaceRepoConfig(workspace, filePath, errors) {
   const repos = workspace.repos;
   const hooks = workspace.hooks;
@@ -358,8 +533,21 @@ function validateWorkspaceRepoConfig(workspace, filePath, errors) {
   if (Array.isArray(repos)) {
     for (const repo of repos) {
       if (!isObject(repo)) continue;
+      const source = repo.source;
       const checkout = repo.checkout;
       const clone = repo.clone;
+      if (isObject(source) && isObject(checkout)) {
+        const sourceType = source.type;
+        const resolve = checkout.resolve;
+        if (sourceType === "local" && typeof resolve === "string") {
+          errors.push({
+            severity: "warning",
+            filePath,
+            location: `workspace.repos[path=${repo.path}]`,
+            message: "checkout.resolve has no effect for a local source. Use source.type to choose where the repo comes from; keep checkout.ref or checkout.ancestor only when pinning a local source."
+          });
+        }
+      }
       if (isObject(checkout) && isObject(clone)) {
         const ancestor = checkout.ancestor;
         const depth = clone.depth;
@@ -491,7 +679,7 @@ function validateMetadata(parsed, filePath, errors) {
   }
 }
 function validateTestsStringPath(testsPath, filePath, errors) {
-  const ext = import_node_path2.default.extname(testsPath);
+  const ext = import_node_path3.default.extname(testsPath);
   if (!VALID_TEST_FILE_EXTENSIONS.has(ext)) {
     errors.push({
       severity: "warning",
@@ -637,12 +825,12 @@ function validateContentForRoleMarkers(content, location, filePath, errors) {
 }
 
 // src/evaluation/validation/targets-validator.ts
-var import_promises3 = require("fs/promises");
-var import_node_path4 = __toESM(require("path"), 1);
-var import_yaml3 = require("yaml");
+var import_promises4 = require("fs/promises");
+var import_node_path5 = __toESM(require("path"), 1);
+var import_yaml4 = require("yaml");
 
 // src/evaluation/providers/targets.ts
-var import_node_path3 = __toESM(require("path"), 1);
+var import_node_path4 = __toESM(require("path"), 1);
 var import_zod = require("zod");
 var CliHealthcheckHttpInputSchema = import_zod.z.object({
   url: import_zod.z.string().min(1, "healthcheck URL is required"),
@@ -722,7 +910,6 @@ var CliTargetConfigSchema = import_zod.z.object({
 var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set([
   "PROMPT",
   "PROMPT_FILE",
-  "GUIDELINES",
   "EVAL_ID",
   "ATTEMPT",
   "FILES",
@@ -1058,11 +1245,11 @@ function validateUnknownSettings(target, provider, absolutePath, location, error
 }
 async function validateTargetsFile(filePath) {
   const errors = [];
-  const absolutePath = import_node_path4.default.resolve(filePath);
+  const absolutePath = import_node_path5.default.resolve(filePath);
   let parsed;
   try {
-    const content = await (0, import_promises3.readFile)(absolutePath, "utf8");
-    parsed = (0, import_yaml3.parse)(content);
+    const content = await (0, import_promises4.readFile)(absolutePath, "utf8");
+    parsed = (0, import_yaml4.parse)(content);
   } catch (error) {
     errors.push({
       severity: "error",
@@ -1261,13 +1448,13 @@ async function validateTargetsFile(filePath) {
 }
 
 // src/evaluation/validation/config-validator.ts
-var import_promises4 = require("fs/promises");
-var import_yaml4 = require("yaml");
+var import_promises5 = require("fs/promises");
+var import_yaml5 = require("yaml");
 async function validateConfigFile(filePath) {
   const errors = [];
   try {
-    const content = await (0, import_promises4.readFile)(filePath, "utf8");
-    const parsed = (0, import_yaml4.parse)(content);
+    const content = await (0, import_promises5.readFile)(filePath, "utf8");
+    const parsed = (0, import_yaml5.parse)(content);
     if (typeof parsed !== "object" || parsed === null) {
       errors.push({
         severity: "error",
@@ -1277,31 +1464,6 @@ async function validateConfigFile(filePath) {
       return { valid: false, filePath, fileType: "config", errors };
     }
     const config = parsed;
-    const guidelinePatterns = config.guideline_patterns;
-    if (guidelinePatterns !== void 0) {
-      if (!Array.isArray(guidelinePatterns)) {
-        errors.push({
-          severity: "error",
-          filePath,
-          location: "guideline_patterns",
-          message: "Field 'guideline_patterns' must be an array"
-        });
-      } else if (!guidelinePatterns.every((p) => typeof p === "string")) {
-        errors.push({
-          severity: "error",
-          filePath,
-          location: "guideline_patterns",
-          message: "All entries in 'guideline_patterns' must be strings"
-        });
-      } else if (guidelinePatterns.length === 0) {
-        errors.push({
-          severity: "warning",
-          filePath,
-          location: "guideline_patterns",
-          message: "Field 'guideline_patterns' is empty. Consider removing it or adding patterns."
-        });
-      }
-    }
     const evalPatterns = config.eval_patterns;
     if (evalPatterns !== void 0) {
       if (!Array.isArray(evalPatterns)) {
@@ -1338,13 +1500,7 @@ async function validateConfigFile(filePath) {
         });
       }
     }
-    const allowedFields = /* @__PURE__ */ new Set([
-      "$schema",
-      "guideline_patterns",
-      "eval_patterns",
-      "required_version",
-      "execution"
-    ]);
+    const allowedFields = /* @__PURE__ */ new Set(["$schema", "eval_patterns", "required_version", "execution"]);
     const unexpectedFields = Object.keys(config).filter((key) => !allowedFields.has(key));
     if (unexpectedFields.length > 0) {
       errors.push({
@@ -1370,31 +1526,31 @@ async function validateConfigFile(filePath) {
 }
 
 // src/evaluation/validation/file-reference-validator.ts
-var import_promises6 = require("fs/promises");
-var import_node_path6 = __toESM(require("path"), 1);
-var import_yaml5 = require("yaml");
+var import_promises7 = require("fs/promises");
+var import_node_path7 = __toESM(require("path"), 1);
+var import_yaml6 = require("yaml");
 
 // src/evaluation/file-utils.ts
 var import_node_fs = require("fs");
-var import_promises5 = require("fs/promises");
-var import_node_path5 = __toESM(require("path"), 1);
+var import_promises6 = require("fs/promises");
+var import_node_path6 = __toESM(require("path"), 1);
 async function fileExists(filePath) {
   try {
-    await (0, import_promises5.access)(filePath, import_node_fs.constants.F_OK);
+    await (0, import_promises6.access)(filePath, import_node_fs.constants.F_OK);
     return true;
   } catch {
     return false;
   }
 }
 async function findGitRoot(startPath) {
-  let currentDir = import_node_path5.default.dirname(import_node_path5.default.resolve(startPath));
-  const root = import_node_path5.default.parse(currentDir).root;
+  let currentDir = import_node_path6.default.dirname(import_node_path6.default.resolve(startPath));
+  const root = import_node_path6.default.parse(currentDir).root;
   while (currentDir !== root) {
-    const gitPath = import_node_path5.default.join(currentDir, ".git");
+    const gitPath = import_node_path6.default.join(currentDir, ".git");
     if (await fileExists(gitPath)) {
       return currentDir;
     }
-    const parentDir = import_node_path5.default.dirname(currentDir);
+    const parentDir = import_node_path6.default.dirname(currentDir);
     if (parentDir === currentDir) {
       break;
     }
@@ -1405,16 +1561,16 @@ async function findGitRoot(startPath) {
 function buildSearchRoots(evalPath, repoRoot) {
   const uniqueRoots = [];
   const addRoot = (root) => {
-    const normalized = import_node_path5.default.resolve(root);
+    const normalized = import_node_path6.default.resolve(root);
     if (!uniqueRoots.includes(normalized)) {
       uniqueRoots.push(normalized);
     }
   };
-  let currentDir = import_node_path5.default.dirname(evalPath);
+  let currentDir = import_node_path6.default.dirname(evalPath);
   let reachedBoundary = false;
   while (!reachedBoundary) {
     addRoot(currentDir);
-    const parentDir = import_node_path5.default.dirname(currentDir);
+    const parentDir = import_node_path6.default.dirname(currentDir);
     if (currentDir === repoRoot || parentDir === currentDir) {
       reachedBoundary = true;
     } else {
@@ -1432,16 +1588,16 @@ function trimLeadingSeparators(value) {
 async function resolveFileReference(rawValue, searchRoots) {
   const displayPath = trimLeadingSeparators(rawValue);
   const potentialPaths = [];
-  if (import_node_path5.default.isAbsolute(rawValue)) {
-    potentialPaths.push(import_node_path5.default.normalize(rawValue));
+  if (import_node_path6.default.isAbsolute(rawValue)) {
+    potentialPaths.push(import_node_path6.default.normalize(rawValue));
   }
   for (const base of searchRoots) {
-    potentialPaths.push(import_node_path5.default.resolve(base, displayPath));
+    potentialPaths.push(import_node_path6.default.resolve(base, displayPath));
   }
   const attempted = [];
   const seen = /* @__PURE__ */ new Set();
   for (const candidate of potentialPaths) {
-    const absoluteCandidate = import_node_path5.default.resolve(candidate);
+    const absoluteCandidate = import_node_path6.default.resolve(candidate);
     if (seen.has(absoluteCandidate)) {
       continue;
     }
@@ -1460,7 +1616,7 @@ function isObject3(value) {
 }
 async function validateFileReferences(evalFilePath) {
   const errors = [];
-  const absolutePath = import_node_path6.default.resolve(evalFilePath);
+  const absolutePath = import_node_path7.default.resolve(evalFilePath);
   const gitRoot = await findGitRoot(absolutePath);
   if (!gitRoot) {
     errors.push({
@@ -1473,8 +1629,8 @@ async function validateFileReferences(evalFilePath) {
   const searchRoots = buildSearchRoots(absolutePath, gitRoot);
   let parsed;
   try {
-    const content = await (0, import_promises6.readFile)(absolutePath, "utf8");
-    parsed = (0, import_yaml5.parse)(content);
+    const content = await (0, import_promises7.readFile)(absolutePath, "utf8");
+    parsed = (0, import_yaml6.parse)(content);
   } catch {
     return errors;
   }
@@ -1561,7 +1717,7 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
         });
       } else {
         try {
-          const fileContent = await (0, import_promises6.readFile)(resolvedPath, "utf8");
+          const fileContent = await (0, import_promises7.readFile)(resolvedPath, "utf8");
           if (fileContent.trim().length === 0) {
             errors.push({
               severity: "warning",