@agentv/core 2.7.1-next.4 → 2.7.1-next.6

package/dist/evaluation/validation/index.cjs

@@ -174,6 +174,20 @@ async function validateEvalFile(filePath) {
     };
   }
   validateMetadata(parsed, absolutePath, errors);
+  const suiteInput = parsed.input;
+  if (suiteInput !== void 0) {
+    if (typeof suiteInput === "string") {
+    } else if (Array.isArray(suiteInput)) {
+      validateMessages(suiteInput, "input", absolutePath, errors);
+    } else {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: "input",
+        message: "Invalid suite-level 'input' field (must be a string or array of messages)"
+      });
+    }
+  }
   let cases = parsed.tests;
   if (cases === void 0 && "eval_cases" in parsed) {
     cases = parsed.eval_cases;
@@ -534,27 +548,25 @@ var import_yaml3 = require("yaml");
 var import_node_path3 = __toESM(require("path"), 1);
 var import_zod = require("zod");
 var CliHealthcheckHttpInputSchema = import_zod.z.object({
-  type: import_zod.z.literal("http"),
   url: import_zod.z.string().min(1, "healthcheck URL is required"),
   timeout_seconds: import_zod.z.number().positive().optional(),
   timeoutSeconds: import_zod.z.number().positive().optional()
 });
 var CliHealthcheckCommandInputSchema = import_zod.z.object({
-  type: import_zod.z.literal("command"),
-  command_template: import_zod.z.string().optional(),
-  commandTemplate: import_zod.z.string().optional(),
+  command: import_zod.z.string().min(1, "healthcheck command is required"),
   cwd: import_zod.z.string().optional(),
   timeout_seconds: import_zod.z.number().positive().optional(),
   timeoutSeconds: import_zod.z.number().positive().optional()
 });
-var CliHealthcheckInputSchema = import_zod.z.discriminatedUnion("type", [
+var CliHealthcheckInputSchema = import_zod.z.union([
   CliHealthcheckHttpInputSchema,
   CliHealthcheckCommandInputSchema
 ]);
 var CliTargetInputSchema = import_zod.z.object({
   name: import_zod.z.string().min(1, "target name is required"),
   provider: import_zod.z.string().refine((p) => p.toLowerCase() === "cli", { message: "provider must be 'cli'" }),
-  // Command template - required (accept both naming conventions)
+  // Command - required (accept both naming conventions)
+  command: import_zod.z.string().optional(),
   command_template: import_zod.z.string().optional(),
   commandTemplate: import_zod.z.string().optional(),
   // Files format - optional
@@ -586,26 +598,27 @@ var CliTargetInputSchema = import_zod.z.object({
   workers: import_zod.z.number().int().min(1).optional(),
   provider_batching: import_zod.z.boolean().optional(),
   providerBatching: import_zod.z.boolean().optional()
-}).refine((data) => data.command_template !== void 0 || data.commandTemplate !== void 0, {
-  message: "Either command_template or commandTemplate is required"
-});
+}).refine(
+  (data) => data.command !== void 0 || data.command_template !== void 0 || data.commandTemplate !== void 0,
+  {
+    message: "'command' is required"
+  }
+);
 var CliHealthcheckHttpSchema = import_zod.z.object({
-  type: import_zod.z.literal("http"),
   url: import_zod.z.string().min(1),
   timeoutMs: import_zod.z.number().positive().optional()
 }).strict();
 var CliHealthcheckCommandSchema = import_zod.z.object({
-  type: import_zod.z.literal("command"),
-  commandTemplate: import_zod.z.string().min(1),
+  command: import_zod.z.string().min(1),
   cwd: import_zod.z.string().optional(),
   timeoutMs: import_zod.z.number().positive().optional()
 }).strict();
-var CliHealthcheckSchema = import_zod.z.discriminatedUnion("type", [
+var CliHealthcheckSchema = import_zod.z.union([
   CliHealthcheckHttpSchema,
   CliHealthcheckCommandSchema
 ]);
 var CliTargetConfigSchema = import_zod.z.object({
-  commandTemplate: import_zod.z.string().min(1),
+  command: import_zod.z.string().min(1),
   filesFormat: import_zod.z.string().optional(),
   cwd: import_zod.z.string().optional(),
   workspaceTemplate: import_zod.z.string().optional(),
@@ -616,6 +629,7 @@ var CliTargetConfigSchema = import_zod.z.object({
 }).strict();
 var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set([
   "PROMPT",
+  "PROMPT_FILE",
   "GUIDELINES",
   "EVAL_ID",
   "ATTEMPT",
@@ -897,21 +911,16 @@ async function validateTargetsFile(filePath) {
     };
   }
   function validateCliSettings(target, absolutePath2, location, errors2) {
-    const commandTemplate = target.command_template ?? target.commandTemplate;
-    if (typeof commandTemplate !== "string" || commandTemplate.trim().length === 0) {
+    const command = target.command ?? target.command_template ?? target.commandTemplate;
+    if (typeof command !== "string" || command.trim().length === 0) {
       errors2.push({
         severity: "error",
         filePath: absolutePath2,
-        location: `${location}.commandTemplate`,
-        message: "CLI provider requires 'command_template' or 'commandTemplate' as a non-empty string"
+        location: `${location}.command`,
+        message: "CLI provider requires 'command' as a non-empty string"
       });
     } else {
-      recordUnknownPlaceholders(
-        commandTemplate,
-        absolutePath2,
-        `${location}.commandTemplate`,
-        errors2
-      );
+      recordUnknownPlaceholders(command, absolutePath2, `${location}.command`, errors2);
     }
     const healthcheck = target.healthcheck;
     if (healthcheck !== void 0) {
@@ -928,16 +937,6 @@ async function validateTargetsFile(filePath) {
       });
       return;
     }
-    const type = healthcheck.type;
-    if (type !== "http" && type !== "command") {
-      errors2.push({
-        severity: "error",
-        filePath: absolutePath2,
-        location: `${location}.type`,
-        message: "healthcheck.type must be either 'http' or 'command'"
-      });
-      return;
-    }
     const timeoutSeconds = healthcheck.timeout_seconds ?? healthcheck.timeoutSeconds;
     if (timeoutSeconds !== void 0) {
       const numericTimeout = Number(timeoutSeconds);
@@ -950,34 +949,26 @@ async function validateTargetsFile(filePath) {
         });
       }
     }
-    if (type === "http") {
-      const url = healthcheck.url;
-      if (typeof url !== "string" || url.trim().length === 0) {
-        errors2.push({
-          severity: "error",
-          filePath: absolutePath2,
-          location: `${location}.url`,
-          message: "healthcheck.url must be a non-empty string for http checks"
-        });
-      }
-      return;
-    }
-    const commandTemplate = healthcheck.command_template ?? healthcheck.commandTemplate;
-    if (typeof commandTemplate !== "string" || commandTemplate.trim().length === 0) {
+    const hasUrl = typeof healthcheck.url === "string" && healthcheck.url.trim().length > 0;
+    const hasCommand = typeof healthcheck.command === "string" && healthcheck.command.trim().length > 0;
+    if (!hasUrl && !hasCommand) {
       errors2.push({
         severity: "error",
         filePath: absolutePath2,
-        location: `${location}.commandTemplate`,
-        message: "healthcheck.commandTemplate must be a non-empty string for command checks"
+        location,
+        message: "healthcheck must have either 'url' (HTTP) or 'command' (command)"
       });
-    } else {
-      recordUnknownPlaceholders(
-        commandTemplate,
-        absolutePath2,
-        `${location}.commandTemplate`,
-        errors2
-      );
+      return;
+    }
+    if (hasUrl) {
+      return;
     }
+    recordUnknownPlaceholders(
+      healthcheck.command,
+      absolutePath2,
+      `${location}.command`,
+      errors2
+    );
     const cwd = healthcheck.cwd;
     if (cwd !== void 0 && typeof cwd !== "string") {
       errors2.push({