agentv 3.13.1 → 3.13.3

package/dist/{chunk-LSXO22CF.js → chunk-PACTPWEN.js}

@@ -27,12 +27,12 @@ import {
   subscribeToCopilotCliLogEntries,
   subscribeToCopilotSdkLogEntries,
   subscribeToPiLogEntries
-} from "./chunk-K747KGDP.js";
+} from "./chunk-D3LNJUUB.js";
 
 // package.json
 var package_default = {
   name: "agentv",
-  version: "3.13.1",
+  version: "3.13.3",
   description: "CLI entry point for AgentV",
   type: "module",
   repository: {
@@ -1644,13 +1644,15 @@ function escapeXml(str) {
 var JunitWriter = class _JunitWriter {
   filePath;
   results = [];
+  threshold;
   closed = false;
-  constructor(filePath) {
+  constructor(filePath, options) {
     this.filePath = filePath;
+    this.threshold = options?.threshold ?? 0.5;
   }
-  static async open(filePath) {
+  static async open(filePath, options) {
     await mkdir5(path9.dirname(filePath), { recursive: true });
-    return new _JunitWriter(filePath);
+    return new _JunitWriter(filePath, options);
   }
   async append(result) {
     if (this.closed) {
@@ -1675,7 +1677,7 @@ var JunitWriter = class _JunitWriter {
     }
     const suiteXmls = [];
     for (const [suiteName, results] of grouped) {
-      const failures = results.filter((r) => r.score < 0.5).length;
+      const failures = results.filter((r) => r.score < this.threshold).length;
       const errors = results.filter((r) => r.error !== void 0).length;
       const testCases = results.map((r) => {
         const time = r.durationMs ? (r.durationMs / 1e3).toFixed(3) : "0.000";
@@ -1684,7 +1686,7 @@ var JunitWriter = class _JunitWriter {
           inner = `
       <error message="${escapeXml(r.error)}">${escapeXml(r.error)}</error>
     `;
-        } else if (r.score < 0.5) {
+        } else if (r.score < this.threshold) {
           const message = `score=${r.score.toFixed(3)}`;
           const failedAssertions = r.assertions.filter((a) => !a.passed);
           const detail = [
@@ -1704,7 +1706,7 @@ ${testCases.join("\n")}
       );
     }
     const totalTests = this.results.length;
-    const totalFailures = this.results.filter((r) => r.score < 0.5).length;
+    const totalFailures = this.results.filter((r) => r.score < this.threshold).length;
     const totalErrors = this.results.filter((r) => r.error !== void 0).length;
     const xml = `<?xml version="1.0" encoding="UTF-8"?>
 <testsuites tests="${totalTests}" failures="${totalFailures}" errors="${totalErrors}">
@@ -1785,7 +1787,7 @@ async function createOutputWriter(filePath, format) {
   }
 }
 var SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([".jsonl", ".json", ".xml", ".yaml", ".yml", ".html", ".htm"]);
-function createWriterFromPath(filePath) {
+function createWriterFromPath(filePath, options) {
   const ext = path11.extname(filePath).toLowerCase();
   switch (ext) {
     case ".jsonl":
@@ -1793,7 +1795,7 @@ function createWriterFromPath(filePath) {
     case ".json":
       return JsonWriter.open(filePath);
     case ".xml":
-      return JunitWriter.open(filePath);
+      return JunitWriter.open(filePath, { threshold: options?.threshold });
     case ".yaml":
     case ".yml":
       return YamlWriter.open(filePath);
@@ -1806,8 +1808,8 @@ function createWriterFromPath(filePath) {
       );
   }
 }
-async function createMultiWriter(filePaths) {
-  const writers = await Promise.all(filePaths.map((fp) => createWriterFromPath(fp)));
+async function createMultiWriter(filePaths, options) {
+  const writers = await Promise.all(filePaths.map((fp) => createWriterFromPath(fp, options)));
   return {
     async append(result) {
       await Promise.all(writers.map((w) => w.append(result)));
@@ -2385,6 +2387,12 @@ function formatMatrixSummary(results) {
   lines.push(`${"Average".padEnd(testIdColWidth)}  ${avgCells.join("  ")}`);
   return lines.join("\n");
 }
+function formatThresholdSummary(meanScore, threshold) {
+  const passed = meanScore >= threshold;
+  const verdict = passed ? "PASS" : "FAIL";
+  const message = `Suite score: ${meanScore.toFixed(2)} (threshold: ${threshold.toFixed(2)}) \u2014 ${verdict}`;
+  return { passed, message };
+}
 
 // ../../packages/core/dist/evaluation/validation/index.js
 import { readFile as readFile3 } from "node:fs/promises";
@@ -4048,7 +4056,8 @@ function normalizeOptions(rawOptions, config, yamlExecution) {
     artifacts: normalizeString(rawOptions.artifacts),
     graderTarget: normalizeString(rawOptions.graderTarget),
     model: normalizeString(rawOptions.model),
-    outputMessages: normalizeOutputMessages(normalizeString(rawOptions.outputMessages))
+    outputMessages: normalizeOutputMessages(normalizeString(rawOptions.outputMessages)),
+    threshold: normalizeOptionalNumber(rawOptions.threshold)
   };
 }
 async function ensureFileExists(filePath, description) {
@@ -4194,7 +4203,8 @@ async function prepareFileMetadata(params) {
     yamlCache: suite.cacheConfig?.enabled,
     yamlCachePath: suite.cacheConfig?.cachePath,
     totalBudgetUsd: suite.totalBudgetUsd,
-    failOnError: suite.failOnError
+    failOnError: suite.failOnError,
+    threshold: suite.threshold
   };
 }
 async function runWithLimit(items, limit, task) {
@@ -4350,6 +4360,9 @@ async function runSingleEvalFile(params) {
 }
 async function runEvalCommand(input) {
   const cwd = process.cwd();
+  if (!process.env.AGENTV_RUN_TIMESTAMP) {
+    process.env.AGENTV_RUN_TIMESTAMP = (/* @__PURE__ */ new Date()).toISOString().replace(/:/g, "-").replace(/\./g, "-");
+  }
   let config = null;
   try {
     config = await loadTsConfig(cwd);
@@ -4408,7 +4421,7 @@ async function runEvalCommand(input) {
   const useFileExport = !!options.otelFile;
   if (options.exportOtel || useFileExport) {
     try {
-      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-LCZDS36N.js");
+      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-KPMR7RBT.js");
       let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
       let headers = {};
       if (options.otelBackend) {
@@ -4454,12 +4467,9 @@ async function runEvalCommand(input) {
   const uniqueOutputPaths = [...new Set(allOutputPaths)];
   const reportedOutputPaths = extraOutputPaths.length > 0 ? [outputPath, ...extraOutputPaths] : [outputPath];
   const uniqueReportedOutputPaths = [...new Set(reportedOutputPaths)];
-  let outputWriter;
   if (uniqueOutputPaths.length === 1) {
-    outputWriter = await createOutputWriter(primaryWritePath, options.format);
     console.log(`Output path: ${outputPath}`);
   } else {
-    outputWriter = await createMultiWriter(uniqueOutputPaths);
     console.log("Output paths:");
     for (const p of uniqueReportedOutputPaths) {
       console.log(`  ${p}`);
@@ -4518,6 +4528,18 @@ async function runEvalCommand(input) {
   if (cacheEnabled) {
     console.log(`Response cache: enabled${yamlCachePath ? ` (${yamlCachePath})` : ""}`);
   }
+  const yamlThreshold = firstMeta?.threshold;
+  const resolvedThreshold = options.threshold ?? yamlThreshold;
+  if (resolvedThreshold !== void 0 && (resolvedThreshold < 0 || resolvedThreshold > 1)) {
+    throw new Error("--threshold must be between 0 and 1");
+  }
+  const writerOptions = resolvedThreshold !== void 0 ? { threshold: resolvedThreshold } : void 0;
+  let outputWriter;
+  if (uniqueOutputPaths.length === 1) {
+    outputWriter = await createOutputWriter(primaryWritePath, options.format);
+  } else {
+    outputWriter = await createMultiWriter(uniqueOutputPaths, writerOptions);
+  }
   const isMatrixMode = Array.from(fileMetadata.values()).some((meta) => meta.selections.length > 1);
   let totalEvalCount = 0;
   for (const meta of fileMetadata.values()) {
@@ -4641,6 +4663,13 @@ async function runEvalCommand(input) {
     }
     const summary = calculateEvaluationSummary(allResults);
     console.log(formatEvaluationSummary(summary));
+    let thresholdFailed = false;
+    if (resolvedThreshold !== void 0) {
+      const thresholdResult = formatThresholdSummary(summary.mean, resolvedThreshold);
+      console.log(`
+${thresholdResult.message}`);
+      thresholdFailed = !thresholdResult.passed;
+    }
     if (isMatrixMode && allResults.length > 0) {
       console.log(formatMatrixSummary(allResults));
     }
@@ -4722,7 +4751,8 @@ Tip: ${summary.executionErrorCount} execution error(s) detected. Re-run failed t
       executionErrorCount: summary.executionErrorCount,
       outputPath,
       testFiles: resolvedTestFiles,
-      target: options.target
+      target: options.target,
+      thresholdFailed
     };
   } finally {
     unsubscribeCodexLogs();
@@ -4781,4 +4811,4 @@ export {
   selectTarget,
   runEvalCommand
 };
-//# sourceMappingURL=chunk-LSXO22CF.js.map
+//# sourceMappingURL=chunk-PACTPWEN.js.map