agentv 2.10.0 → 2.11.1

package/dist/{chunk-PC3FAOHT.js → chunk-IL7CRMY6.js}

@@ -10,6 +10,7 @@ import {
   findGitRoot,
   isEvaluatorKind,
   listTargetNames,
+  loadConfig,
   loadTestSuite,
   loadTsConfig,
   normalizeLineEndings,
@@ -21,9 +22,10 @@ import {
   shouldEnableCache,
   shouldSkipCacheForTemperature,
   subscribeToCodexLogEntries,
+  subscribeToCopilotCliLogEntries,
   subscribeToCopilotSdkLogEntries,
   subscribeToPiLogEntries
-} from "./chunk-RJWTL3VS.js";
+} from "./chunk-MQIQH5LB.js";
 
 // src/commands/eval/shared.ts
 import { constants } from "node:fs";
@@ -1063,7 +1065,20 @@ function inferFileTypeFromPath(filePath) {
   }
   return "eval";
 }
-var ASSERTION_TYPES_WITH_VALUE = /* @__PURE__ */ new Set(["contains", "equals", "regex"]);
+var ASSERTION_TYPES_WITH_STRING_VALUE = /* @__PURE__ */ new Set([
+  "contains",
+  "icontains",
+  "starts_with",
+  "ends_with",
+  "equals",
+  "regex"
+]);
+var ASSERTION_TYPES_WITH_ARRAY_VALUE = /* @__PURE__ */ new Set([
+  "contains_any",
+  "contains_all",
+  "icontains_any",
+  "icontains_all"
+]);
 var VALID_TEST_FILE_EXTENSIONS = /* @__PURE__ */ new Set([".yaml", ".yml", ".jsonl"]);
 var NAME_PATTERN = /^[a-z0-9-]+$/;
 function isObject(value) {
@@ -1246,6 +1261,9 @@ async function validateEvalFile(filePath) {
       validateAssertArray(assertField, location, absolutePath, errors);
     }
   }
+  if (isObject(parsed.workspace)) {
+    validateWorkspaceRepoConfig(parsed.workspace, absolutePath, errors);
+  }
   return {
     valid: errors.filter((e) => e.severity === "error").length === 0,
     filePath: absolutePath,
@@ -1253,6 +1271,48 @@ async function validateEvalFile(filePath) {
     errors
   };
 }
+function validateWorkspaceRepoConfig(workspace, filePath, errors) {
+  const repos = workspace.repos;
+  const reset = workspace.reset;
+  const isolation = workspace.isolation;
+  if (Array.isArray(repos)) {
+    for (const repo of repos) {
+      if (!isObject(repo)) continue;
+      const checkout = repo.checkout;
+      const clone = repo.clone;
+      if (isObject(checkout) && isObject(clone)) {
+        const ancestor = checkout.ancestor;
+        const depth = clone.depth;
+        if (typeof ancestor === "number" && typeof depth === "number" && depth < ancestor + 1) {
+          errors.push({
+            severity: "warning",
+            filePath,
+            location: `workspace.repos[path=${repo.path}]`,
+            message: `clone.depth (${depth}) may be insufficient for checkout.ancestor (${ancestor}). Recommend depth >= ${ancestor + 1}.`
+          });
+        }
+      }
+    }
+  }
+  if (isObject(reset) && reset.strategy && reset.strategy !== "none") {
+    if (!Array.isArray(repos) || repos.length === 0) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location: "workspace.reset",
+        message: `reset.strategy '${reset.strategy}' has no effect without repos.`
+      });
+    }
+  }
+  if (isObject(reset) && reset.after_each === true && isolation === "per_test") {
+    errors.push({
+      severity: "warning",
+      filePath,
+      location: "workspace.reset",
+      message: "reset.after_each is redundant with isolation: per_test (each test gets a fresh workspace)."
+    });
+  }
+}
 function validateMessages(messages, location, filePath, errors) {
   for (let i = 0; i < messages.length; i++) {
     const message = messages[i];
@@ -1402,7 +1462,7 @@ function validateAssertArray(assertField, parentLocation, filePath, errors) {
       });
       continue;
     }
-    if (ASSERTION_TYPES_WITH_VALUE.has(typeValue)) {
+    if (ASSERTION_TYPES_WITH_STRING_VALUE.has(typeValue)) {
       const value = item.value;
       if (value === void 0 || typeof value !== "string") {
         errors.push({
@@ -1426,6 +1486,18 @@ function validateAssertArray(assertField, parentLocation, filePath, errors) {
         }
       }
     }
+    if (ASSERTION_TYPES_WITH_ARRAY_VALUE.has(typeValue)) {
+      const value = item.value;
+      if (!Array.isArray(value) || value.length === 0) {
+        errors.push({
+          severity: "warning",
+          filePath,
+          location: `${location}.value`,
+          message: `Assertion type '${typeValue}' requires a 'value' field (non-empty string array).`
+        });
+        continue;
+      }
+    }
     const required = item.required;
     if (required !== void 0) {
       validateRequiredField(required, location, filePath, errors);
@@ -1571,6 +1643,26 @@ var COPILOT_SDK_SETTINGS = /* @__PURE__ */ new Set([
   "workspace_template",
   "workspaceTemplate"
 ]);
+var COPILOT_CLI_SETTINGS = /* @__PURE__ */ new Set([
+  ...COMMON_SETTINGS,
+  "executable",
+  "command",
+  "binary",
+  "args",
+  "arguments",
+  "model",
+  "cwd",
+  "timeout_seconds",
+  "timeoutSeconds",
+  "log_dir",
+  "logDir",
+  "log_format",
+  "logFormat",
+  "system_prompt",
+  "systemPrompt",
+  "workspace_template",
+  "workspaceTemplate"
+]);
 var VSCODE_SETTINGS = /* @__PURE__ */ new Set([
   ...COMMON_SETTINGS,
   "executable",
@@ -1631,11 +1723,12 @@ function getKnownSettings(provider) {
     case "codex":
     case "codex-cli":
       return CODEX_SETTINGS;
-    case "copilot":
     case "copilot-sdk":
     case "copilot_sdk":
-    case "copilot-cli":
       return COPILOT_SDK_SETTINGS;
+    case "copilot":
+    case "copilot-cli":
+      return COPILOT_CLI_SETTINGS;
     case "claude":
     case "claude-code":
     case "claude-sdk":
@@ -2307,6 +2400,13 @@ function normalizeString(value) {
   const trimmed = value.trim();
   return trimmed.length > 0 ? trimmed : void 0;
 }
+function resolveTimestampPlaceholder(value) {
+  if (!value.includes("{timestamp}")) {
+    return value;
+  }
+  const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+  return value.replaceAll("{timestamp}", timestamp);
+}
 function normalizeNumber(value, fallback) {
   if (typeof value === "number" && Number.isFinite(value)) {
     return value;
@@ -2331,7 +2431,7 @@ function normalizeOptionalNumber(value) {
   }
   return void 0;
 }
-function normalizeOptions(rawOptions, config) {
+function normalizeOptions(rawOptions, config, yamlExecution) {
   const cliFormat = normalizeString(rawOptions.outputFormat);
   const configFormat = config?.output?.format;
   const formatStr = cliFormat ?? configFormat ?? "jsonl";
@@ -2382,12 +2482,14 @@ function normalizeOptions(rawOptions, config) {
     maxRetries: cliMaxRetries ?? configMaxRetries ?? 2,
     cache: resolvedCache,
     noCache: resolvedNoCache,
-    verbose: normalizeBoolean(rawOptions.verbose),
-    keepWorkspaces: normalizeBoolean(rawOptions.keepWorkspaces),
+    // Boolean OR: config `true` cannot be overridden to `false` from CLI.
+    // Intentional — there are no --no-verbose / --no-keep-workspaces flags.
+    verbose: normalizeBoolean(rawOptions.verbose) || yamlExecution?.verbose === true,
+    keepWorkspaces: normalizeBoolean(rawOptions.keepWorkspaces) || yamlExecution?.keep_workspaces === true,
     cleanupWorkspaces: normalizeBoolean(rawOptions.cleanupWorkspaces),
     trace: normalizeBoolean(rawOptions.trace),
-    otelFile: normalizeString(rawOptions.otelFile),
-    traceFile: normalizeString(rawOptions.traceFile),
+    otelFile: normalizeString(rawOptions.otelFile) ?? (yamlExecution?.otel_file ? resolveTimestampPlaceholder(yamlExecution.otel_file) : void 0),
+    traceFile: normalizeString(rawOptions.traceFile) ?? (yamlExecution?.trace_file ? resolveTimestampPlaceholder(yamlExecution.trace_file) : void 0),
     exportOtel: normalizeBoolean(rawOptions.exportOtel),
     otelBackend: normalizeString(rawOptions.otelBackend),
     otelCaptureContent: normalizeBoolean(rawOptions.otelCaptureContent),
@@ -2672,8 +2774,9 @@ async function runEvalCommand(input) {
       `Warning: Failed to load agentv config: ${err instanceof Error ? err.message : String(err)}`
     );
   }
-  const options = normalizeOptions(input.rawOptions, config);
   const repoRoot = await findRepoRoot(cwd);
+  const yamlConfig = await loadConfig(path10.join(cwd, "_"), repoRoot);
+  const options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution);
   if (options.keepWorkspaces && options.cleanupWorkspaces) {
     console.warn(
       "Warning: Both --keep-workspaces and --cleanup-workspaces specified. --cleanup-workspaces takes precedence."
@@ -2686,7 +2789,7 @@ async function runEvalCommand(input) {
   const useFileExport = !!(options.otelFile || options.traceFile);
   if (options.exportOtel || useFileExport) {
     try {
-      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-BGRU67HI.js");
+      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-OVEHXEXC.js");
       let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
       let headers = {};
       if (options.otelBackend) {
@@ -2814,7 +2917,14 @@ async function runEvalCommand(input) {
     progressReporter.addLogPaths([entry.filePath], "pi");
   });
   const seenCopilotLogPaths = /* @__PURE__ */ new Set();
-  const unsubscribeCopilotLogs = subscribeToCopilotSdkLogEntries((entry) => {
+  const unsubscribeCopilotSdkLogs = subscribeToCopilotSdkLogEntries((entry) => {
+    if (!entry.filePath || seenCopilotLogPaths.has(entry.filePath)) {
+      return;
+    }
+    seenCopilotLogPaths.add(entry.filePath);
+    progressReporter.addLogPaths([entry.filePath], "copilot");
+  });
+  const unsubscribeCopilotCliLogs = subscribeToCopilotCliLogEntries((entry) => {
     if (!entry.filePath || seenCopilotLogPaths.has(entry.filePath)) {
       return;
     }
@@ -2908,7 +3018,8 @@ Results written to: ${outputPath}`);
   } finally {
     unsubscribeCodexLogs();
     unsubscribePiLogs();
-    unsubscribeCopilotLogs();
+    unsubscribeCopilotSdkLogs();
+    unsubscribeCopilotCliLogs();
     await outputWriter.close().catch(() => void 0);
     if (otelExporter) {
       try {
@@ -2949,4 +3060,4 @@ export {
   selectTarget,
   runEvalCommand
 };
-//# sourceMappingURL=chunk-PC3FAOHT.js.map
+//# sourceMappingURL=chunk-IL7CRMY6.js.map