agentv 2.11.2 → 2.12.0

package/dist/{chunk-IL7CRMY6.js → chunk-YBJX5CP6.js}

@@ -25,7 +25,7 @@ import {
   subscribeToCopilotCliLogEntries,
   subscribeToCopilotSdkLogEntries,
   subscribeToPiLogEntries
-} from "./chunk-MQIQH5LB.js";
+} from "./chunk-LUHCYBMD.js";
 
 // src/commands/eval/shared.ts
 import { constants } from "node:fs";
@@ -872,7 +872,6 @@ function buildHistogram(values) {
   return bins;
 }
 function calculateEvaluationSummary(results) {
-  const scores = results.map((result) => result.score);
   const total = results.length;
   const errors = results.filter((result) => result.error !== void 0).map((result) => ({ testId: result.testId, error: result.error }));
   const errorCount = errors.length;
@@ -888,18 +887,39 @@ function calculateEvaluationSummary(results) {
       topResults: [],
       bottomResults: [],
       errorCount: 0,
-      errors: []
+      errors: [],
+      executionErrorCount: 0,
+      qualityFailureCount: 0,
+      passedCount: 0,
+      byFailureStage: {},
+      byFailureReason: {}
     };
   }
-  const mean = computeMean(scores);
-  const median = computeMedian(scores);
-  const min = Math.min(...scores);
-  const max = Math.max(...scores);
-  const standardDeviation = computeStandardDeviation(scores);
-  const histogram = buildHistogram(scores);
-  const sortedResults = [...results].sort((a, b) => b.score - a.score);
+  const executionErrors = results.filter((r) => r.executionStatus === "execution_error");
+  const qualityResults = results.filter((r) => r.executionStatus !== "execution_error");
+  const qualityScores = qualityResults.map((r) => r.score);
+  const mean = computeMean(qualityScores);
+  const median = computeMedian(qualityScores);
+  const min = qualityScores.length > 0 ? Math.min(...qualityScores) : 0;
+  const max = qualityScores.length > 0 ? Math.max(...qualityScores) : 0;
+  const standardDeviation = computeStandardDeviation(qualityScores);
+  const histogram = buildHistogram(qualityScores);
+  const sortedResults = [...qualityResults].sort((a, b) => b.score - a.score);
   const topResults = sortedResults.slice(0, Math.min(3, sortedResults.length));
   const bottomResults = sortedResults.slice(-Math.min(3, sortedResults.length));
+  const executionErrorCount = executionErrors.length;
+  const qualityFailureCount = results.filter((r) => r.executionStatus === "quality_failure").length;
+  const passedCount = results.filter((r) => r.executionStatus === "ok").length;
+  const byFailureStage = {};
+  const byFailureReason = {};
+  for (const result of executionErrors) {
+    if (result.failureStage) {
+      byFailureStage[result.failureStage] = (byFailureStage[result.failureStage] ?? 0) + 1;
+    }
+    if (result.failureReasonCode) {
+      byFailureReason[result.failureReasonCode] = (byFailureReason[result.failureReasonCode] ?? 0) + 1;
+    }
+  }
   return {
     total,
     mean,
@@ -911,7 +931,12 @@ function calculateEvaluationSummary(results) {
     topResults,
     bottomResults,
     errorCount,
-    errors
+    errors,
+    executionErrorCount,
+    qualityFailureCount,
+    passedCount,
+    byFailureStage,
+    byFailureReason
   };
 }
 function formatScore(value) {
@@ -924,7 +949,7 @@ function formatEvaluationSummary(summary) {
   const lines = [];
   if (summary.errorCount > 0) {
     lines.push("\n==================================================");
-    lines.push("ERRORS");
+    lines.push("EXECUTION ERRORS");
     lines.push("==================================================");
     for (const error of summary.errors) {
       lines.push(`
@@ -937,11 +962,21 @@ function formatEvaluationSummary(summary) {
   lines.push("EVALUATION SUMMARY");
   lines.push("==================================================");
   lines.push(`Total tests: ${summary.total}`);
-  if (summary.errorCount > 0) {
-    lines.push(`Failed: ${summary.errorCount}`);
-    lines.push(`Passed: ${summary.total - summary.errorCount}`);
+  lines.push(`Passed: ${summary.passedCount}`);
+  if (summary.qualityFailureCount > 0) {
+    lines.push(`Quality failures: ${summary.qualityFailureCount}`);
+  }
+  if (summary.executionErrorCount > 0) {
+    lines.push(`Execution errors: ${summary.executionErrorCount}`);
+  }
+  if (summary.executionErrorCount > 0) {
+    const qualityCount = summary.total - summary.executionErrorCount;
+    lines.push(
+      `Mean score: ${formatScore(summary.mean)} (${qualityCount} quality tests, ${summary.executionErrorCount} execution errors excluded)`
+    );
+  } else {
+    lines.push(`Mean score: ${formatScore(summary.mean)}`);
   }
-  lines.push(`Mean score: ${formatScore(summary.mean)}`);
   lines.push(`Median score: ${formatScore(summary.median)}`);
   lines.push(`Min score: ${formatScore(summary.min)}`);
   lines.push(`Max score: ${formatScore(summary.max)}`);
@@ -961,6 +996,20 @@ function formatEvaluationSummary(summary) {
   summary.bottomResults.forEach((result, index) => {
     lines.push(`  ${index + 1}. ${result.testId}: ${formatScore(result.score)}`);
   });
+  const failureStageEntries = Object.entries(summary.byFailureStage);
+  if (failureStageEntries.length > 0) {
+    lines.push("\nExecution errors by stage:");
+    for (const [stage, count] of failureStageEntries) {
+      lines.push(`  ${stage}: ${count}`);
+    }
+  }
+  const failureReasonEntries = Object.entries(summary.byFailureReason);
+  if (failureReasonEntries.length > 0) {
+    lines.push("\nExecution errors by reason:");
+    for (const [reason, count] of failureReasonEntries) {
+      lines.push(`  ${reason}: ${count}`);
+    }
+  }
   return lines.join("\n");
 }
 function formatMatrixSummary(results) {
@@ -2484,12 +2533,13 @@ function normalizeOptions(rawOptions, config, yamlExecution) {
     noCache: resolvedNoCache,
     // Boolean OR: config `true` cannot be overridden to `false` from CLI.
     // Intentional — there are no --no-verbose / --no-keep-workspaces flags.
-    verbose: normalizeBoolean(rawOptions.verbose) || yamlExecution?.verbose === true,
-    keepWorkspaces: normalizeBoolean(rawOptions.keepWorkspaces) || yamlExecution?.keep_workspaces === true,
+    // Precedence: CLI > YAML config > TS config
+    verbose: normalizeBoolean(rawOptions.verbose) || yamlExecution?.verbose === true || config?.execution?.verbose === true,
+    keepWorkspaces: normalizeBoolean(rawOptions.keepWorkspaces) || yamlExecution?.keep_workspaces === true || config?.execution?.keepWorkspaces === true,
     cleanupWorkspaces: normalizeBoolean(rawOptions.cleanupWorkspaces),
-    trace: normalizeBoolean(rawOptions.trace),
-    otelFile: normalizeString(rawOptions.otelFile) ?? (yamlExecution?.otel_file ? resolveTimestampPlaceholder(yamlExecution.otel_file) : void 0),
-    traceFile: normalizeString(rawOptions.traceFile) ?? (yamlExecution?.trace_file ? resolveTimestampPlaceholder(yamlExecution.trace_file) : void 0),
+    // Precedence: CLI > YAML config > TS config
+    otelFile: normalizeString(rawOptions.otelFile) ?? (yamlExecution?.otel_file ? resolveTimestampPlaceholder(yamlExecution.otel_file) : void 0) ?? (config?.execution?.otelFile ? resolveTimestampPlaceholder(config.execution.otelFile) : void 0),
+    traceFile: normalizeString(rawOptions.traceFile) ?? (yamlExecution?.trace_file ? resolveTimestampPlaceholder(yamlExecution.trace_file) : void 0) ?? (config?.execution?.traceFile ? resolveTimestampPlaceholder(config.execution.traceFile) : void 0),
     exportOtel: normalizeBoolean(rawOptions.exportOtel),
     otelBackend: normalizeString(rawOptions.otelBackend),
     otelCaptureContent: normalizeBoolean(rawOptions.otelCaptureContent),
@@ -2789,7 +2839,7 @@ async function runEvalCommand(input) {
   const useFileExport = !!(options.otelFile || options.traceFile);
   if (options.exportOtel || useFileExport) {
     try {
-      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-OVEHXEXC.js");
+      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-OPPA4P5R.js");
       let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
       let headers = {};
       if (options.otelBackend) {
@@ -3060,4 +3110,4 @@ export {
   selectTarget,
   runEvalCommand
 };
-//# sourceMappingURL=chunk-IL7CRMY6.js.map
+//# sourceMappingURL=chunk-YBJX5CP6.js.map