@agentv/core 0.7.5 → 0.10.0

package/dist/evaluation/validation/index.cjs

@@ -44,7 +44,7 @@ module.exports = __toCommonJS(validation_exports);
 var import_promises = require("fs/promises");
 var import_yaml = require("yaml");
 var SCHEMA_EVAL_V2 = "agentv-eval-v2";
-var SCHEMA_TARGETS_V2 = "agentv-targets-v2.1";
+var SCHEMA_TARGETS_V2 = "agentv-targets-v2.2";
 var SCHEMA_CONFIG_V2 = "agentv-config-v2";
 async function detectFileType(filePath) {
   try {
@@ -196,14 +196,14 @@ async function validateEvalFile(filePath) {
       validateMessages(inputMessages, `${location}.input_messages`, absolutePath, errors);
     }
     const expectedMessages = evalCase["expected_messages"];
-    if (!Array.isArray(expectedMessages)) {
+    if (expectedMessages !== void 0 && !Array.isArray(expectedMessages)) {
       errors.push({
         severity: "error",
         filePath: absolutePath,
         location: `${location}.expected_messages`,
-        message: "Missing or invalid 'expected_messages' field (must be an array)"
+        message: "Invalid 'expected_messages' field (must be an array if provided)"
       });
-    } else {
+    } else if (Array.isArray(expectedMessages)) {
       validateMessages(expectedMessages, `${location}.expected_messages`, absolutePath, errors);
     }
   }
@@ -239,11 +239,13 @@ function validateMessages(messages, location, filePath, errors) {
     }
     const content = message["content"];
     if (typeof content === "string") {
+      validateContentForRoleMarkers(content, `${msgLocation}.content`, filePath, errors);
     } else if (Array.isArray(content)) {
       for (let j = 0; j < content.length; j++) {
         const contentItem = content[j];
         const contentLocation = `${msgLocation}.content[${j}]`;
         if (typeof contentItem === "string") {
+          validateContentForRoleMarkers(contentItem, contentLocation, filePath, errors);
         } else if (isObject(contentItem)) {
           const type = contentItem["type"];
           if (typeof type !== "string") {
@@ -263,6 +265,8 @@ function validateMessages(messages, location, filePath, errors) {
                 location: `${contentLocation}.value`,
                 message: "Content with type 'text' must have a 'value' field"
               });
+            } else {
+              validateContentForRoleMarkers(value, `${contentLocation}.value`, filePath, errors);
             }
           }
         } else {
@@ -284,12 +288,35 @@ function validateMessages(messages, location, filePath, errors) {
     }
   }
 }
+function validateContentForRoleMarkers(content, location, filePath, errors) {
+  const markers = ["@[System]:", "@[User]:", "@[Assistant]:", "@[Tool]:"];
+  for (const marker of markers) {
+    if (content.toLowerCase().includes(marker.toLowerCase())) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location,
+        message: `Content contains potential role marker '${marker}'. This may confuse agentic providers or cause prompt injection.`
+      });
+    }
+  }
+}
 
 // src/evaluation/validation/targets-validator.ts
 var import_promises3 = require("fs/promises");
 var import_node_path2 = __toESM(require("path"), 1);
 var import_yaml3 = require("yaml");
 
+// src/evaluation/providers/targets.ts
+var import_zod = require("zod");
+var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID", "ATTEMPT", "FILES", "OUTPUT_FILE"]);
+var BASE_TARGET_SCHEMA = import_zod.z.object({
+  name: import_zod.z.string().min(1, "target name is required"),
+  provider: import_zod.z.string().min(1, "provider is required"),
+  judge_target: import_zod.z.string().optional(),
+  workers: import_zod.z.number().int().min(1).optional()
+}).passthrough();
+
 // src/evaluation/providers/types.ts
 var KNOWN_PROVIDERS = [
   "azure",
@@ -317,18 +344,7 @@ var PROVIDER_ALIASES = [
   "vertex"
   // legacy/future support
 ];
-var TARGETS_SCHEMA_V2 = "agentv-targets-v2.1";
-
-// src/evaluation/providers/targets.ts
-var import_zod = require("zod");
-var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID", "ATTEMPT", "FILES", "OUTPUT_FILE"]);
-var BASE_TARGET_SCHEMA = import_zod.z.object({
-  name: import_zod.z.string().min(1, "target name is required"),
-  provider: import_zod.z.string().min(1, "provider is required"),
-  settings: import_zod.z.record(import_zod.z.unknown()).optional(),
-  judge_target: import_zod.z.string().optional(),
-  workers: import_zod.z.number().int().min(1).optional()
-});
+var TARGETS_SCHEMA_V2 = "agentv-targets-v2.2";
 
 // src/evaluation/validation/targets-validator.ts
 function isObject2(value) {
@@ -338,8 +354,21 @@ var COMMON_SETTINGS = /* @__PURE__ */ new Set([
   "provider_batching",
   "providerBatching"
 ]);
+var RETRY_SETTINGS = /* @__PURE__ */ new Set([
+  "max_retries",
+  "maxRetries",
+  "retry_initial_delay_ms",
+  "retryInitialDelayMs",
+  "retry_max_delay_ms",
+  "retryMaxDelayMs",
+  "retry_backoff_factor",
+  "retryBackoffFactor",
+  "retry_status_codes",
+  "retryStatusCodes"
+]);
 var AZURE_SETTINGS = /* @__PURE__ */ new Set([
   ...COMMON_SETTINGS,
+  ...RETRY_SETTINGS,
   "endpoint",
   "resource",
   "resourceName",
@@ -356,6 +385,7 @@ var AZURE_SETTINGS = /* @__PURE__ */ new Set([
 ]);
 var ANTHROPIC_SETTINGS = /* @__PURE__ */ new Set([
   ...COMMON_SETTINGS,
+  ...RETRY_SETTINGS,
   "api_key",
   "apiKey",
   "model",
@@ -369,6 +399,7 @@ var ANTHROPIC_SETTINGS = /* @__PURE__ */ new Set([
 ]);
 var GEMINI_SETTINGS = /* @__PURE__ */ new Set([
   ...COMMON_SETTINGS,
+  ...RETRY_SETTINGS,
   "api_key",
   "apiKey",
   "model",
@@ -456,13 +487,14 @@ function getKnownSettings(provider) {
       return null;
   }
 }
-function validateUnknownSettings(settings, provider, absolutePath, location, errors) {
+function validateUnknownSettings(target, provider, absolutePath, location, errors) {
   const knownSettings = getKnownSettings(provider);
   if (!knownSettings) {
     return;
   }
-  for (const key of Object.keys(settings)) {
-    if (!knownSettings.has(key)) {
+  const baseFields = /* @__PURE__ */ new Set(["name", "provider", "judge_target", "workers", "$schema", "targets"]);
+  for (const key of Object.keys(target)) {
+    if (!baseFields.has(key) && !knownSettings.has(key)) {
       errors.push({
         severity: "warning",
         filePath: absolutePath,
@@ -492,17 +524,8 @@ async function validateTargetsFile(filePath) {
       errors
     };
   }
-  function validateCliSettings(settings, absolutePath2, location, errors2) {
-    if (!isObject2(settings)) {
-      errors2.push({
-        severity: "error",
-        filePath: absolutePath2,
-        location,
-        message: "CLI provider requires a 'settings' object"
-      });
-      return;
-    }
-    const commandTemplate = settings["command_template"] ?? settings["commandTemplate"];
+  function validateCliSettings(target, absolutePath2, location, errors2) {
+    const commandTemplate = target["command_template"] ?? target["commandTemplate"];
     if (typeof commandTemplate !== "string" || commandTemplate.trim().length === 0) {
       errors2.push({
         severity: "error",
@@ -513,7 +536,7 @@ async function validateTargetsFile(filePath) {
     } else {
       recordUnknownPlaceholders(commandTemplate, absolutePath2, `${location}.commandTemplate`, errors2);
     }
-    const attachmentsFormat = settings["attachments_format"] ?? settings["attachmentsFormat"];
+    const attachmentsFormat = target["attachments_format"] ?? target["attachmentsFormat"];
     if (attachmentsFormat !== void 0 && typeof attachmentsFormat !== "string") {
       errors2.push({
         severity: "error",
@@ -522,7 +545,7 @@ async function validateTargetsFile(filePath) {
         message: "'attachmentsFormat' must be a string when provided"
       });
     }
-    const filesFormat = settings["files_format"] ?? settings["filesFormat"];
+    const filesFormat = target["files_format"] ?? target["filesFormat"];
     if (filesFormat !== void 0 && typeof filesFormat !== "string") {
       errors2.push({
         severity: "error",
@@ -531,7 +554,7 @@ async function validateTargetsFile(filePath) {
         message: "'filesFormat' must be a string when provided"
       });
     }
-    const cwd = settings["cwd"];
+    const cwd = target["cwd"];
     if (cwd !== void 0 && typeof cwd !== "string") {
       errors2.push({
         severity: "error",
@@ -540,7 +563,7 @@ async function validateTargetsFile(filePath) {
         message: "'cwd' must be a string when provided"
       });
     }
-    const timeoutSeconds = settings["timeout_seconds"] ?? settings["timeoutSeconds"];
+    const timeoutSeconds = target["timeout_seconds"] ?? target["timeoutSeconds"];
     if (timeoutSeconds !== void 0) {
       const numericTimeout = Number(timeoutSeconds);
       if (!Number.isFinite(numericTimeout) || numericTimeout <= 0) {
@@ -552,29 +575,7 @@ async function validateTargetsFile(filePath) {
         });
       }
     }
-    const envOverrides = settings["env"];
-    if (envOverrides !== void 0) {
-      if (!isObject2(envOverrides)) {
-        errors2.push({
-          severity: "error",
-          filePath: absolutePath2,
-          location: `${location}.env`,
-          message: "'env' must be an object with string values"
-        });
-      } else {
-        for (const [key, value] of Object.entries(envOverrides)) {
-          if (typeof value !== "string" || value.trim().length === 0) {
-            errors2.push({
-              severity: "error",
-              filePath: absolutePath2,
-              location: `${location}.env.${key}`,
-              message: `Environment override '${key}' must be a non-empty string`
-            });
-          }
-        }
-      }
-    }
-    const healthcheck = settings["healthcheck"];
+    const healthcheck = target["healthcheck"];
     if (healthcheck !== void 0) {
       validateCliHealthcheck(healthcheck, absolutePath2, `${location}.healthcheck`, errors2);
     }
@@ -745,20 +746,11 @@ async function validateTargetsFile(filePath) {
         message: `Unknown provider '${provider}'. Known providers: ${knownProviders.join(", ")}`
       });
     }
-    const settings = target["settings"];
-    if (providerValue !== "cli" && settings !== void 0 && !isObject2(settings)) {
-      errors.push({
-        severity: "error",
-        filePath: absolutePath,
-        location: `${location}.settings`,
-        message: "Invalid 'settings' field (must be an object)"
-      });
-    }
     if (providerValue === "cli") {
-      validateCliSettings(settings, absolutePath, `${location}.settings`, errors);
+      validateCliSettings(target, absolutePath, location, errors);
     }
-    if (settings !== void 0 && isObject2(settings) && typeof provider === "string") {
-      validateUnknownSettings(settings, provider, absolutePath, `${location}.settings`, errors);
+    if (typeof provider === "string") {
+      validateUnknownSettings(target, provider, absolutePath, location, errors);
     }
     const judgeTarget = target["judge_target"];
     if (judgeTarget !== void 0 && typeof judgeTarget !== "string") {