agentv 4.5.2 → 4.6.0

package/dist/{chunk-OIBYQMCK.js → chunk-5GZJIXTY.js}

@@ -29,12 +29,12 @@ import {
   subscribeToCopilotCliLogEntries,
   subscribeToCopilotSdkLogEntries,
   subscribeToPiLogEntries
-} from "./chunk-7DRAXDVC.js";
+} from "./chunk-KQQTEWZF.js";
 
 // package.json
 var package_default = {
   name: "agentv",
-  version: "4.5.2",
+  version: "4.6.0",
   description: "CLI entry point for AgentV",
   type: "module",
   repository: {
@@ -102,9 +102,21 @@ async function resolveEvalPaths(evalPaths, cwd) {
   if (normalizedInputs.length === 0) {
     throw new Error("No eval paths provided.");
   }
+  const includePatterns = [];
+  const ignorePatterns = [];
+  for (const input of normalizedInputs) {
+    if (input.startsWith("!")) {
+      ignorePatterns.push(input.slice(1));
+    } else {
+      includePatterns.push(input);
+    }
+  }
+  if (includePatterns.length === 0) {
+    throw new Error("No eval paths provided (only negation patterns found).");
+  }
   const unmatched = [];
   const results = /* @__PURE__ */ new Set();
-  for (const pattern of normalizedInputs) {
+  for (const pattern of includePatterns) {
     const candidatePath = path.isAbsolute(pattern) ? path.normalize(pattern) : path.resolve(cwd, pattern);
     try {
       const stats = await stat(candidatePath);
@@ -119,7 +131,8 @@ async function resolveEvalPaths(evalPaths, cwd) {
           onlyFiles: true,
           unique: true,
           dot: true,
-          followSymbolicLinks: true
+          followSymbolicLinks: true,
+          ignore: ignorePatterns
         });
         if (dirMatches.length === 0) {
           unmatched.push(pattern);
@@ -139,7 +152,8 @@ async function resolveEvalPaths(evalPaths, cwd) {
       onlyFiles: true,
       unique: true,
       dot: true,
-      followSymbolicLinks: true
+      followSymbolicLinks: true,
+      ignore: ignorePatterns
     });
     const yamlMatches = matches.filter((filePath) => /\.(ya?ml|jsonl|json)$/i.test(filePath));
     if (yamlMatches.length === 0) {
@@ -3114,6 +3128,8 @@ var OPENAI_SETTINGS = /* @__PURE__ */ new Set([
   "model",
   "deployment",
   "variant",
+  "api_format",
+  "apiFormat",
   "temperature",
   "max_output_tokens",
   "maxTokens"
@@ -3506,16 +3522,17 @@ async function validateTargetsFile(filePath) {
       });
     }
     const provider = target.provider;
+    const hasUseTarget = typeof target.use_target === "string" && target.use_target.trim().length > 0;
     const providerValue = typeof provider === "string" ? provider.trim().toLowerCase() : void 0;
     const isTemplated = typeof provider === "string" && /^\$\{\{.+\}\}$/.test(provider.trim());
-    if (typeof provider !== "string" || provider.trim().length === 0) {
+    if (!hasUseTarget && (typeof provider !== "string" || provider.trim().length === 0)) {
       errors.push({
         severity: "error",
         filePath: absolutePath,
         location: `${location}.provider`,
-        message: "Missing or invalid 'provider' field (must be a non-empty string)"
+        message: "Missing or invalid 'provider' field (must be a non-empty string, or use use_target for delegation)"
       });
-    } else if (!isTemplated && !knownProviders.includes(provider)) {
+    } else if (typeof provider === "string" && !isTemplated && !knownProviders.includes(provider)) {
       errors.push({
         severity: "warning",
         filePath: absolutePath,
@@ -3761,6 +3778,34 @@ var ANSI_RESET3 = "\x1B[0m";
 function isTTY() {
   return process.stdout.isTTY ?? false;
 }
+function resolveUseTarget(name, definitions, env, targetsFilePath) {
+  const maxDepth = 5;
+  let current = definitions.find((d) => d.name === name);
+  if (!current) {
+    const available = listTargetNames(definitions).join(", ");
+    throw new Error(
+      `Target '${name}' not found in ${targetsFilePath}. Available targets: ${available}`
+    );
+  }
+  for (let depth = 0; depth < maxDepth; depth++) {
+    const useTarget = current.use_target;
+    if (useTarget === void 0 || useTarget === null) break;
+    const raw = String(useTarget).trim();
+    if (raw.length === 0) break;
+    const envMatch = raw.match(/^\$\{\{\s*([A-Z0-9_]+)\s*\}\}$/i);
+    const resolved = envMatch ? env[envMatch[1]] ?? "" : raw;
+    if (resolved.trim().length === 0) break;
+    const next = definitions.find((d) => d.name === resolved.trim());
+    if (!next) {
+      const available = listTargetNames(definitions).join(", ");
+      throw new Error(
+        `Target '${name}' use_target '${resolved.trim()}' not found in ${targetsFilePath}. Available targets: ${available}`
+      );
+    }
+    current = next;
+  }
+  return current;
+}
 async function readTestSuiteTarget(testFilePath) {
   const metadata = await readTestSuiteMetadata(testFilePath);
   return metadata.target;
@@ -3824,15 +3869,7 @@ Errors in ${targetsFilePath}:`);
   const definitions = await readTargetDefinitions(targetsFilePath);
   const fileTargetName = await readTestSuiteTarget(testFilePath);
   const targetChoice = pickTargetName({ cliTargetName, fileTargetName });
-  const targetDefinition = definitions.find(
-    (definition) => definition.name === targetChoice.name
-  );
-  if (!targetDefinition) {
-    const available = listTargetNames(definitions).join(", ");
-    throw new Error(
-      `Target '${targetChoice.name}' not found in ${targetsFilePath}. Available targets: ${available}`
-    );
-  }
+  const targetDefinition = resolveUseTarget(targetChoice.name, definitions, env, targetsFilePath);
   if (dryRun) {
     const mockTarget = {
       kind: "mock",
@@ -3915,15 +3952,7 @@ Errors in ${targetsFilePath}:`);
   const definitions = await readTargetDefinitions(targetsFilePath);
   const results = [];
   for (const name of targetNames) {
-    const targetDefinition = definitions.find(
-      (definition) => definition.name === name
-    );
-    if (!targetDefinition) {
-      const available = listTargetNames(definitions).join(", ");
-      throw new Error(
-        `Target '${name}' not found in ${targetsFilePath}. Available targets: ${available}`
-      );
-    }
+    const targetDefinition = resolveUseTarget(name, definitions, env, targetsFilePath);
     if (dryRun) {
       const mockTarget = {
         kind: "mock",
@@ -4008,6 +4037,24 @@ function normalizeOptionalNumber(value) {
 function normalizeWorkspaceMode(value) {
   return value === "pooled" || value === "temp" || value === "static" ? value : void 0;
 }
+function normalizeStringArray(value) {
+  if (Array.isArray(value)) {
+    return value.filter((v) => typeof v === "string" && v.trim().length > 0);
+  }
+  return [];
+}
+function matchesTagFilters(fileTags, includeTags, excludeTags) {
+  const tags = new Set(fileTags ?? []);
+  if (includeTags.length > 0) {
+    for (const required of includeTags) {
+      if (!tags.has(required)) return false;
+    }
+  }
+  for (const excluded of excludeTags) {
+    if (tags.has(excluded)) return false;
+  }
+  return true;
+}
 function normalizeOutputMessages(cliValue) {
   if (cliValue === void 0) {
     return 1;
@@ -4116,7 +4163,9 @@ function normalizeOptions(rawOptions, config, yamlExecution) {
     graderTarget: normalizeString(rawOptions.graderTarget),
     model: normalizeString(rawOptions.model),
     outputMessages: normalizeOutputMessages(normalizeString(rawOptions.outputMessages)),
-    threshold: normalizeOptionalNumber(rawOptions.threshold)
+    threshold: normalizeOptionalNumber(rawOptions.threshold),
+    tags: normalizeStringArray(rawOptions.tag),
+    excludeTags: normalizeStringArray(rawOptions.excludeTag)
   };
 }
 async function ensureFileExists(filePath, description) {
@@ -4266,7 +4315,8 @@ async function prepareFileMetadata(params) {
     yamlCachePath: suite.cacheConfig?.cachePath,
     totalBudgetUsd: suite.totalBudgetUsd,
     failOnError: suite.failOnError,
-    threshold: suite.threshold
+    threshold: suite.threshold,
+    tags: suite.metadata?.tags
   };
 }
 async function runWithLimit(items, limit, task) {
@@ -4484,7 +4534,7 @@ async function runEvalCommand(input) {
   const useFileExport = !!options.otelFile;
   if (options.exportOtel || useFileExport) {
     try {
-      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-VWMHFUXR.js");
+      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-FBPCDLOY.js");
       let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
       let headers = {};
       if (options.otelBackend) {
@@ -4578,6 +4628,25 @@ async function runEvalCommand(input) {
     });
     fileMetadata.set(testFilePath, meta);
   }
+  const hasTagFilters = options.tags.length > 0 || options.excludeTags.length > 0;
+  if (hasTagFilters) {
+    const skippedFiles = [];
+    for (const [testFilePath, meta] of fileMetadata.entries()) {
+      if (!matchesTagFilters(meta.tags, options.tags, options.excludeTags)) {
+        fileMetadata.delete(testFilePath);
+        skippedFiles.push(path15.relative(cwd, testFilePath));
+      }
+    }
+    if (skippedFiles.length > 0 && options.verbose) {
+      console.log(
+        `Skipped ${skippedFiles.length} eval file(s) by tag filter: ${skippedFiles.join(", ")}`
+      );
+    }
+    if (fileMetadata.size === 0) {
+      console.log("No eval files matched the tag filters. Nothing to run.");
+      return;
+    }
+  }
   const firstMeta = fileMetadata.values().next().value;
   const yamlCacheEnabled = firstMeta?.yamlCache;
   const yamlCachePath = firstMeta?.yamlCachePath;
@@ -4667,8 +4736,9 @@ async function runEvalCommand(input) {
       }
     }
   }
+  const activeTestFiles = resolvedTestFiles.filter((f) => fileMetadata.has(f));
   try {
-    await runWithLimit(resolvedTestFiles, fileConcurrency, async (testFilePath) => {
+    await runWithLimit(activeTestFiles, fileConcurrency, async (testFilePath) => {
       const targetPrep = fileMetadata.get(testFilePath);
       if (!targetPrep) {
         throw new Error(`Missing metadata for ${testFilePath}`);
@@ -4685,30 +4755,56 @@ async function runEvalCommand(input) {
           if (applicableEvalCases.length === 0) {
             return [];
           }
-          const result = await runSingleEvalFile({
-            testFilePath,
-            cwd,
-            repoRoot,
-            options,
-            outputWriter,
-            otelExporter,
-            cache,
-            evaluationRunner,
-            workersOverride: perFileWorkers,
-            yamlWorkers: targetPrep.yamlWorkers,
-            progressReporter,
-            seenEvalCases,
-            displayIdTracker,
-            selection,
-            inlineTargetLabel,
-            evalCases: applicableEvalCases,
-            trialsConfig: targetPrep.trialsConfig,
-            matrixMode: targetPrep.selections.length > 1,
-            totalBudgetUsd: targetPrep.totalBudgetUsd,
-            failOnError: targetPrep.failOnError,
-            threshold: resolvedThreshold
-          });
-          return result.results;
+          try {
+            const result = await runSingleEvalFile({
+              testFilePath,
+              cwd,
+              repoRoot,
+              options,
+              outputWriter,
+              otelExporter,
+              cache,
+              evaluationRunner,
+              workersOverride: perFileWorkers,
+              yamlWorkers: targetPrep.yamlWorkers,
+              progressReporter,
+              seenEvalCases,
+              displayIdTracker,
+              selection,
+              inlineTargetLabel,
+              evalCases: applicableEvalCases,
+              trialsConfig: targetPrep.trialsConfig,
+              matrixMode: targetPrep.selections.length > 1,
+              totalBudgetUsd: targetPrep.totalBudgetUsd,
+              failOnError: targetPrep.failOnError,
+              threshold: resolvedThreshold
+            });
+            return result.results;
+          } catch (fileError) {
+            const message = fileError instanceof Error ? fileError.message : String(fileError);
+            console.error(`
+\u26A0 Eval file failed: ${path15.basename(testFilePath)} \u2014 ${message}
+`);
+            const errorResults = applicableEvalCases.map((evalCase) => ({
+              timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+              testId: evalCase.id,
+              score: 0,
+              assertions: [],
+              output: [],
+              scores: [],
+              error: message,
+              executionStatus: "execution_error",
+              failureStage: "setup",
+              failureReasonCode: "setup_error",
+              durationMs: 0,
+              tokenUsage: { input: 0, output: 0, inputTokens: 0, outputTokens: 0 },
+              target: selection.targetName
+            }));
+            for (const errResult of errorResults) {
+              await outputWriter.append(errResult);
+            }
+            return errorResults;
+          }
         })
       );
       for (const results of targetResults) {
@@ -4738,7 +4834,7 @@ async function runEvalCommand(input) {
       console.log(`Benchmark written to: ${benchmarkPath}`);
     }
     if (usesDefaultArtifactWorkspace) {
-      const evalFile = resolvedTestFiles.length === 1 ? resolvedTestFiles[0] : "";
+      const evalFile = activeTestFiles.length === 1 ? activeTestFiles[0] : "";
       const workspaceDir = path15.dirname(outputPath);
       const {
         testArtifactDir,
@@ -4758,7 +4854,7 @@ async function runEvalCommand(input) {
     }
     if (options.artifacts) {
       const artifactsDir = path15.resolve(options.artifacts);
-      const evalFile = resolvedTestFiles.length === 1 ? resolvedTestFiles[0] : "";
+      const evalFile = activeTestFiles.length === 1 ? activeTestFiles[0] : "";
       const {
         testArtifactDir,
         indexPath,
@@ -4797,7 +4893,7 @@ Results written to: ${outputPath}`);
       await saveRunCache(cwd, outputPath).catch(() => void 0);
     }
     if (summary.executionErrorCount > 0 && !options.retryErrors) {
-      const evalFileArgs = resolvedTestFiles.map((f) => path15.relative(cwd, f)).join(" ");
+      const evalFileArgs = activeTestFiles.map((f) => path15.relative(cwd, f)).join(" ");
       const targetFlag = options.target ? ` --target ${options.target}` : "";
       const relativeOutputPath = path15.relative(cwd, outputPath);
       console.log(
@@ -4809,7 +4905,7 @@ Tip: ${summary.executionErrorCount} execution error(s) detected. Re-run failed t
     return {
       executionErrorCount: summary.executionErrorCount,
       outputPath,
-      testFiles: resolvedTestFiles,
+      testFiles: activeTestFiles,
       target: options.target,
       thresholdFailed
     };
@@ -4872,4 +4968,4 @@ export {
   selectTarget,
   runEvalCommand
 };
-//# sourceMappingURL=chunk-OIBYQMCK.js.map
+//# sourceMappingURL=chunk-5GZJIXTY.js.map