@agentv/core 0.2.3

package/dist/evaluation/validation/index.cjs

@@ -0,0 +1,617 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/evaluation/validation/index.ts
+var validation_exports = {};
+__export(validation_exports, {
+  detectFileType: () => detectFileType,
+  getExpectedSchema: () => getExpectedSchema,
+  isValidSchema: () => isValidSchema,
+  validateEvalFile: () => validateEvalFile,
+  validateFileReferences: () => validateFileReferences,
+  validateTargetsFile: () => validateTargetsFile
+});
+module.exports = __toCommonJS(validation_exports);
+
+// src/evaluation/validation/file-type.ts
+var import_promises = require("fs/promises");
+var import_yaml = require("yaml");
+var SCHEMA_EVAL_V2 = "agentv-eval-v2";
+var SCHEMA_TARGETS_V2 = "agentv-targets-v2";
+async function detectFileType(filePath) {
+  try {
+    const content = await (0, import_promises.readFile)(filePath, "utf8");
+    const parsed = (0, import_yaml.parse)(content);
+    if (typeof parsed !== "object" || parsed === null) {
+      return "unknown";
+    }
+    const record = parsed;
+    const schema = record["$schema"];
+    if (typeof schema !== "string") {
+      return "unknown";
+    }
+    switch (schema) {
+      case SCHEMA_EVAL_V2:
+        return "eval";
+      case SCHEMA_TARGETS_V2:
+        return "targets";
+      default:
+        return "unknown";
+    }
+  } catch {
+    return "unknown";
+  }
+}
+function isValidSchema(schema) {
+  return schema === SCHEMA_EVAL_V2 || schema === SCHEMA_TARGETS_V2;
+}
+function getExpectedSchema(fileType) {
+  switch (fileType) {
+    case "eval":
+      return SCHEMA_EVAL_V2;
+    case "targets":
+      return SCHEMA_TARGETS_V2;
+    default:
+      return void 0;
+  }
+}
+
+// src/evaluation/validation/eval-validator.ts
+var import_promises2 = require("fs/promises");
+var import_node_path = __toESM(require("path"), 1);
+var import_yaml2 = require("yaml");
+var SCHEMA_EVAL_V22 = "agentv-eval-v2";
+function isObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+async function validateEvalFile(filePath) {
+  const errors = [];
+  const absolutePath = import_node_path.default.resolve(filePath);
+  let parsed;
+  try {
+    const content = await (0, import_promises2.readFile)(absolutePath, "utf8");
+    parsed = (0, import_yaml2.parse)(content);
+  } catch (error) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      message: `Failed to parse YAML: ${error.message}`
+    });
+    return {
+      valid: false,
+      filePath: absolutePath,
+      fileType: "eval",
+      errors
+    };
+  }
+  if (!isObject(parsed)) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      message: "File must contain a YAML object"
+    });
+    return {
+      valid: false,
+      filePath: absolutePath,
+      fileType: "eval",
+      errors
+    };
+  }
+  const schema = parsed["$schema"];
+  if (schema !== SCHEMA_EVAL_V22) {
+    const message = typeof schema === "string" ? `Invalid $schema value '${schema}'. Expected '${SCHEMA_EVAL_V22}'` : `Missing required field '$schema'. Expected '${SCHEMA_EVAL_V22}'`;
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      location: "$schema",
+      message
+    });
+  }
+  const evalcases = parsed["evalcases"];
+  if (!Array.isArray(evalcases)) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      location: "evalcases",
+      message: "Missing or invalid 'evalcases' field (must be an array)"
+    });
+    return {
+      valid: errors.length === 0,
+      filePath: absolutePath,
+      fileType: "eval",
+      errors
+    };
+  }
+  for (let i = 0; i < evalcases.length; i++) {
+    const evalCase = evalcases[i];
+    const location = `evalcases[${i}]`;
+    if (!isObject(evalCase)) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location,
+        message: "Eval case must be an object"
+      });
+      continue;
+    }
+    const id = evalCase["id"];
+    if (typeof id !== "string" || id.trim().length === 0) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.id`,
+        message: "Missing or invalid 'id' field (must be a non-empty string)"
+      });
+    }
+    const outcome = evalCase["outcome"];
+    if (typeof outcome !== "string" || outcome.trim().length === 0) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.outcome`,
+        message: "Missing or invalid 'outcome' field (must be a non-empty string)"
+      });
+    }
+    const inputMessages = evalCase["input_messages"];
+    if (!Array.isArray(inputMessages)) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.input_messages`,
+        message: "Missing or invalid 'input_messages' field (must be an array)"
+      });
+    } else {
+      validateMessages(inputMessages, `${location}.input_messages`, absolutePath, errors);
+    }
+    const expectedMessages = evalCase["expected_messages"];
+    if (!Array.isArray(expectedMessages)) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.expected_messages`,
+        message: "Missing or invalid 'expected_messages' field (must be an array)"
+      });
+    } else {
+      validateMessages(expectedMessages, `${location}.expected_messages`, absolutePath, errors);
+    }
+  }
+  return {
+    valid: errors.length === 0,
+    filePath: absolutePath,
+    fileType: "eval",
+    errors
+  };
+}
+function validateMessages(messages, location, filePath, errors) {
+  for (let i = 0; i < messages.length; i++) {
+    const message = messages[i];
+    const msgLocation = `${location}[${i}]`;
+    if (!isObject(message)) {
+      errors.push({
+        severity: "error",
+        filePath,
+        location: msgLocation,
+        message: "Message must be an object"
+      });
+      continue;
+    }
+    const role = message["role"];
+    const validRoles = ["system", "user", "assistant"];
+    if (!validRoles.includes(role)) {
+      errors.push({
+        severity: "error",
+        filePath,
+        location: `${msgLocation}.role`,
+        message: `Invalid role '${role}'. Must be one of: ${validRoles.join(", ")}`
+      });
+    }
+    const content = message["content"];
+    if (typeof content === "string") {
+    } else if (Array.isArray(content)) {
+      for (let j = 0; j < content.length; j++) {
+        const contentItem = content[j];
+        const contentLocation = `${msgLocation}.content[${j}]`;
+        if (typeof contentItem === "string") {
+        } else if (isObject(contentItem)) {
+          const type = contentItem["type"];
+          if (typeof type !== "string") {
+            errors.push({
+              severity: "error",
+              filePath,
+              location: `${contentLocation}.type`,
+              message: "Content object must have a 'type' field"
+            });
+          }
+          if (type === "text") {
+            const value = contentItem["value"];
+            if (typeof value !== "string") {
+              errors.push({
+                severity: "error",
+                filePath,
+                location: `${contentLocation}.value`,
+                message: "Content with type 'text' must have a 'value' field"
+              });
+            }
+          }
+        } else {
+          errors.push({
+            severity: "error",
+            filePath,
+            location: contentLocation,
+            message: "Content array items must be strings or objects"
+          });
+        }
+      }
+    } else {
+      errors.push({
+        severity: "error",
+        filePath,
+        location: `${msgLocation}.content`,
+        message: "Missing or invalid 'content' field (must be a string or array)"
+      });
+    }
+  }
+}
+
+// src/evaluation/validation/targets-validator.ts
+var import_promises3 = require("fs/promises");
+var import_node_path2 = __toESM(require("path"), 1);
+var import_yaml3 = require("yaml");
+var SCHEMA_TARGETS_V22 = "agentv-targets-v2";
+function isObject2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+async function validateTargetsFile(filePath) {
+  const errors = [];
+  const absolutePath = import_node_path2.default.resolve(filePath);
+  let parsed;
+  try {
+    const content = await (0, import_promises3.readFile)(absolutePath, "utf8");
+    parsed = (0, import_yaml3.parse)(content);
+  } catch (error) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      message: `Failed to parse YAML: ${error.message}`
+    });
+    return {
+      valid: false,
+      filePath: absolutePath,
+      fileType: "targets",
+      errors
+    };
+  }
+  if (!isObject2(parsed)) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      message: "File must contain a YAML object"
+    });
+    return {
+      valid: false,
+      filePath: absolutePath,
+      fileType: "targets",
+      errors
+    };
+  }
+  const schema = parsed["$schema"];
+  if (schema !== SCHEMA_TARGETS_V22) {
+    const message = typeof schema === "string" ? `Invalid $schema value '${schema}'. Expected '${SCHEMA_TARGETS_V22}'` : `Missing required field '$schema'. Expected '${SCHEMA_TARGETS_V22}'`;
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      location: "$schema",
+      message
+    });
+  }
+  const targets = parsed["targets"];
+  if (!Array.isArray(targets)) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      location: "targets",
+      message: "Missing or invalid 'targets' field (must be an array)"
+    });
+    return {
+      valid: errors.length === 0,
+      filePath: absolutePath,
+      fileType: "targets",
+      errors
+    };
+  }
+  const knownProviders = ["azure", "openai", "anthropic", "bedrock", "vertex"];
+  for (let i = 0; i < targets.length; i++) {
+    const target = targets[i];
+    const location = `targets[${i}]`;
+    if (!isObject2(target)) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location,
+        message: "Target must be an object"
+      });
+      continue;
+    }
+    const name = target["name"];
+    if (typeof name !== "string" || name.trim().length === 0) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.name`,
+        message: "Missing or invalid 'name' field (must be a non-empty string)"
+      });
+    }
+    const provider = target["provider"];
+    if (typeof provider !== "string" || provider.trim().length === 0) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.provider`,
+        message: "Missing or invalid 'provider' field (must be a non-empty string)"
+      });
+    } else if (!knownProviders.includes(provider)) {
+      errors.push({
+        severity: "warning",
+        filePath: absolutePath,
+        location: `${location}.provider`,
+        message: `Unknown provider '${provider}'. Known providers: ${knownProviders.join(", ")}`
+      });
+    }
+    const settings = target["settings"];
+    if (settings !== void 0 && !isObject2(settings)) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.settings`,
+        message: "Invalid 'settings' field (must be an object)"
+      });
+    }
+    const judgeTarget = target["judge_target"];
+    if (judgeTarget !== void 0 && typeof judgeTarget !== "string") {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.judge_target`,
+        message: "Invalid 'judge_target' field (must be a string)"
+      });
+    }
+  }
+  return {
+    valid: errors.filter((e) => e.severity === "error").length === 0,
+    filePath: absolutePath,
+    fileType: "targets",
+    errors
+  };
+}
+
+// src/evaluation/validation/file-reference-validator.ts
+var import_promises5 = require("fs/promises");
+var import_node_path4 = __toESM(require("path"), 1);
+var import_yaml4 = require("yaml");
+
+// src/evaluation/file-utils.ts
+var import_node_fs = require("fs");
+var import_promises4 = require("fs/promises");
+var import_node_path3 = __toESM(require("path"), 1);
+async function fileExists(filePath) {
+  try {
+    await (0, import_promises4.access)(filePath, import_node_fs.constants.F_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+async function findGitRoot(startPath) {
+  let currentDir = import_node_path3.default.dirname(import_node_path3.default.resolve(startPath));
+  const root = import_node_path3.default.parse(currentDir).root;
+  while (currentDir !== root) {
+    const gitPath = import_node_path3.default.join(currentDir, ".git");
+    if (await fileExists(gitPath)) {
+      return currentDir;
+    }
+    const parentDir = import_node_path3.default.dirname(currentDir);
+    if (parentDir === currentDir) {
+      break;
+    }
+    currentDir = parentDir;
+  }
+  return null;
+}
+function buildSearchRoots(evalPath, repoRoot) {
+  const uniqueRoots = [];
+  const addRoot = (root) => {
+    const normalized = import_node_path3.default.resolve(root);
+    if (!uniqueRoots.includes(normalized)) {
+      uniqueRoots.push(normalized);
+    }
+  };
+  let currentDir = import_node_path3.default.dirname(evalPath);
+  let reachedBoundary = false;
+  while (!reachedBoundary) {
+    addRoot(currentDir);
+    const parentDir = import_node_path3.default.dirname(currentDir);
+    if (currentDir === repoRoot || parentDir === currentDir) {
+      reachedBoundary = true;
+    } else {
+      currentDir = parentDir;
+    }
+  }
+  addRoot(repoRoot);
+  addRoot(process.cwd());
+  return uniqueRoots;
+}
+function trimLeadingSeparators(value) {
+  const trimmed = value.replace(/^[/\\]+/, "");
+  return trimmed.length > 0 ? trimmed : value;
+}
+async function resolveFileReference(rawValue, searchRoots) {
+  const displayPath = trimLeadingSeparators(rawValue);
+  const potentialPaths = [];
+  if (import_node_path3.default.isAbsolute(rawValue)) {
+    potentialPaths.push(import_node_path3.default.normalize(rawValue));
+  }
+  for (const base of searchRoots) {
+    potentialPaths.push(import_node_path3.default.resolve(base, displayPath));
+  }
+  const attempted = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const candidate of potentialPaths) {
+    const absoluteCandidate = import_node_path3.default.resolve(candidate);
+    if (seen.has(absoluteCandidate)) {
+      continue;
+    }
+    seen.add(absoluteCandidate);
+    attempted.push(absoluteCandidate);
+    if (await fileExists(absoluteCandidate)) {
+      return { displayPath, resolvedPath: absoluteCandidate, attempted };
+    }
+  }
+  return { displayPath, attempted };
+}
+
+// src/evaluation/validation/file-reference-validator.ts
+function isObject3(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+async function validateFileReferences(evalFilePath) {
+  const errors = [];
+  const absolutePath = import_node_path4.default.resolve(evalFilePath);
+  const gitRoot = await findGitRoot(absolutePath);
+  if (!gitRoot) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      message: "Cannot validate file references: git repository root not found"
+    });
+    return errors;
+  }
+  const searchRoots = buildSearchRoots(absolutePath, gitRoot);
+  let parsed;
+  try {
+    const content = await (0, import_promises5.readFile)(absolutePath, "utf8");
+    parsed = (0, import_yaml4.parse)(content);
+  } catch {
+    return errors;
+  }
+  if (!isObject3(parsed)) {
+    return errors;
+  }
+  const evalcases = parsed["evalcases"];
+  if (!Array.isArray(evalcases)) {
+    return errors;
+  }
+  for (let i = 0; i < evalcases.length; i++) {
+    const evalCase = evalcases[i];
+    if (!isObject3(evalCase)) {
+      continue;
+    }
+    const inputMessages = evalCase["input_messages"];
+    if (Array.isArray(inputMessages)) {
+      await validateMessagesFileRefs(inputMessages, `evalcases[${i}].input_messages`, searchRoots, absolutePath, errors);
+    }
+    const expectedMessages = evalCase["expected_messages"];
+    if (Array.isArray(expectedMessages)) {
+      await validateMessagesFileRefs(expectedMessages, `evalcases[${i}].expected_messages`, searchRoots, absolutePath, errors);
+    }
+  }
+  return errors;
+}
+async function validateMessagesFileRefs(messages, location, searchRoots, filePath, errors) {
+  for (let i = 0; i < messages.length; i++) {
+    const message = messages[i];
+    if (!isObject3(message)) {
+      continue;
+    }
+    const content = message["content"];
+    if (typeof content === "string") {
+      continue;
+    }
+    if (!Array.isArray(content)) {
+      continue;
+    }
+    for (let j = 0; j < content.length; j++) {
+      const contentItem = content[j];
+      if (!isObject3(contentItem)) {
+        continue;
+      }
+      const type = contentItem["type"];
+      if (type !== "file") {
+        continue;
+      }
+      const value = contentItem["value"];
+      if (typeof value !== "string") {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: `${location}[${i}].content[${j}].value`,
+          message: "File reference must have a 'value' field with the file path"
+        });
+        continue;
+      }
+      const { resolvedPath } = await resolveFileReference(value, searchRoots);
+      if (!resolvedPath) {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: `${location}[${i}].content[${j}]`,
+          message: `Referenced file not found: ${value}`
+        });
+      } else {
+        try {
+          const fileContent = await (0, import_promises5.readFile)(resolvedPath, "utf8");
+          if (fileContent.trim().length === 0) {
+            errors.push({
+              severity: "warning",
+              filePath,
+              location: `${location}[${i}].content[${j}]`,
+              message: `Referenced file is empty: ${value}`
+            });
+          }
+        } catch (error) {
+          errors.push({
+            severity: "error",
+            filePath,
+            location: `${location}[${i}].content[${j}]`,
+            message: `Cannot read referenced file: ${value} (${error.message})`
+          });
+        }
+      }
+    }
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  detectFileType,
+  getExpectedSchema,
+  isValidSchema,
+  validateEvalFile,
+  validateFileReferences,
+  validateTargetsFile
+});
+//# sourceMappingURL=index.cjs.map