agentv 3.3.0 → 3.4.0

package/dist/{chunk-4ZMSAQWS.js → chunk-RE5I3U2S.js}

@@ -25,12 +25,12 @@ import {
   subscribeToCopilotCliLogEntries,
   subscribeToCopilotSdkLogEntries,
   subscribeToPiLogEntries
-} from "./chunk-5M3K2DMV.js";
+} from "./chunk-GOZV2HN2.js";
 
 // package.json
 var package_default = {
   name: "agentv",
-  version: "3.3.0",
+  version: "3.4.0",
   description: "CLI entry point for AgentV",
   type: "module",
   repository: {
@@ -320,19 +320,19 @@ function parseWorkspaceChanges(fileChanges) {
     diff_summary: diffSummary
   };
 }
-function buildExpectations(result) {
-  const expectations = [];
+function buildAssertions(result) {
+  const assertions = [];
   if (result.scores && result.scores.length > 0) {
     for (const evaluator of result.scores) {
       for (const hit of evaluator.hits) {
-        expectations.push({
+        assertions.push({
           text: hit,
           passed: true,
           evidence: evaluator.reasoning ?? ""
         });
       }
       for (const miss of evaluator.misses) {
-        expectations.push({
+        assertions.push({
           text: miss,
           passed: false,
           evidence: evaluator.reasoning ?? ""
@@ -341,13 +341,13 @@ function buildExpectations(result) {
     }
   } else {
     for (const hit of result.hits) {
-      expectations.push({ text: hit, passed: true, evidence: result.reasoning ?? "" });
+      assertions.push({ text: hit, passed: true, evidence: result.reasoning ?? "" });
     }
     for (const miss of result.misses) {
-      expectations.push({ text: miss, passed: false, evidence: result.reasoning ?? "" });
+      assertions.push({ text: miss, passed: false, evidence: result.reasoning ?? "" });
     }
   }
-  return expectations;
+  return assertions;
 }
 function buildEvaluators(scores) {
   if (!scores || scores.length === 0) {
@@ -366,14 +366,14 @@ function buildEvaluators(scores) {
   }));
 }
 function buildGradingArtifact(result) {
-  const expectations = buildExpectations(result);
-  const passed = expectations.filter((e) => e.passed).length;
-  const failed = expectations.filter((e) => !e.passed).length;
-  const total = expectations.length;
+  const assertions = buildAssertions(result);
+  const passed = assertions.filter((e) => e.passed).length;
+  const failed = assertions.filter((e) => !e.passed).length;
+  const total = assertions.length;
   const { toolCalls, total: totalToolCalls } = countToolCalls(result);
   const errorsEncountered = result.error ? 1 : 0;
   return {
-    expectations,
+    assertions,
     summary: {
       passed,
       failed,
@@ -496,6 +496,42 @@ function buildBenchmarkArtifact(results, evalFile = "") {
     notes
   };
 }
+function toCamelCase(str) {
+  return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
+}
+function toCamelCaseDeep(obj) {
+  if (obj === null || obj === void 0) {
+    return obj;
+  }
+  if (Array.isArray(obj)) {
+    return obj.map((item) => toCamelCaseDeep(item));
+  }
+  if (typeof obj === "object") {
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[toCamelCase(key)] = toCamelCaseDeep(value);
+    }
+    return result;
+  }
+  return obj;
+}
+function parseJsonlResults(content) {
+  const results = [];
+  const lines = content.split("\n");
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed.length === 0) {
+      continue;
+    }
+    try {
+      const parsed = JSON.parse(trimmed);
+      const camelCased = toCamelCaseDeep(parsed);
+      results.push(camelCased);
+    } catch {
+    }
+  }
+  return results;
+}
 async function writeArtifactsFromResults(results, outputDir, options) {
   const gradingDir = path3.join(outputDir, "grading");
   const timingPath = path3.join(outputDir, "timing.json");
@@ -1673,6 +1709,24 @@ async function createMultiWriter(filePaths) {
 }
 
 // src/commands/eval/progress-display.ts
+var ANSI_BOLD = "\x1B[1m";
+var ANSI_GREEN = "\x1B[32m";
+var ANSI_RED2 = "\x1B[31m";
+var ANSI_YELLOW2 = "\x1B[33m";
+var ANSI_RESET2 = "\x1B[0m";
+function useColors() {
+  if (process.env.NO_COLOR !== void 0) return false;
+  return process.stdout.isTTY ?? false;
+}
+function formatVerdict(score, verdict) {
+  if (verdict === void 0) return "";
+  const colors = useColors();
+  const scoreStr = score !== void 0 ? score.toFixed(3) : "";
+  const verdictLabel = verdict === "ERROR" ? "ERROR" : `${scoreStr} ${verdict}`;
+  if (!colors) return ` | ${verdictLabel}`;
+  const color = verdict === "PASS" ? ANSI_GREEN : verdict === "FAIL" ? ANSI_RED2 : ANSI_YELLOW2;
+  return ` | ${color}${ANSI_BOLD}${verdictLabel}${ANSI_RESET2}`;
+}
 var ProgressDisplay = class {
   workers = /* @__PURE__ */ new Map();
   totalTests = 0;
@@ -1716,11 +1770,13 @@ var ProgressDisplay = class {
         }
         break;
       case "completed":
-        console.log(`${countPrefix}   \u2705 ${progress.testId}${targetSuffix}`);
+        console.log(
+          `${countPrefix}   \u2705 ${progress.testId}${targetSuffix}${formatVerdict(progress.score, progress.verdict)}`
+        );
         break;
       case "failed":
         console.log(
-          `${countPrefix}   \u274C ${progress.testId}${targetSuffix}${progress.error ? `: ${progress.error}` : ""}`
+          `${countPrefix}   \u274C ${progress.testId}${targetSuffix}${formatVerdict(progress.score, progress.verdict)}${progress.error ? `: ${progress.error}` : ""}`
         );
         break;
     }
@@ -1760,6 +1816,22 @@ var ProgressDisplay = class {
 // src/commands/eval/retry-errors.ts
 import { createReadStream } from "node:fs";
 import { createInterface } from "node:readline";
+function getTestId(result) {
+  return result.testId ?? result.test_id;
+}
+function getExecutionStatus(result) {
+  return result.executionStatus ?? result.execution_status;
+}
+function toEvaluationResult(result) {
+  if (result.testId !== void 0 && result.executionStatus !== void 0) {
+    return result;
+  }
+  return {
+    ...result,
+    testId: getTestId(result) ?? "",
+    executionStatus: getExecutionStatus(result)
+  };
+}
 async function loadErrorTestIds(jsonlPath) {
   const ids = [];
   const rl = createInterface({
@@ -1771,8 +1843,10 @@ async function loadErrorTestIds(jsonlPath) {
     if (!trimmed) continue;
     try {
       const parsed = JSON.parse(trimmed);
-      if (parsed.executionStatus === "execution_error" && parsed.testId) {
-        ids.push(parsed.testId);
+      const executionStatus = getExecutionStatus(parsed);
+      const testId = getTestId(parsed);
+      if (executionStatus === "execution_error" && testId) {
+        ids.push(testId);
       }
     } catch {
     }
@@ -1790,9 +1864,11 @@ async function loadNonErrorResults(jsonlPath) {
     if (!trimmed) continue;
     try {
       const parsed = JSON.parse(trimmed);
-      if (!parsed.testId || parsed.score === void 0) continue;
-      if (parsed.executionStatus !== "execution_error") {
-        results.push(parsed);
+      const testId = getTestId(parsed);
+      const executionStatus = getExecutionStatus(parsed);
+      if (!testId || parsed.score === void 0) continue;
+      if (executionStatus !== "execution_error") {
+        results.push(toEvaluationResult(parsed));
       }
     } catch {
     }
@@ -1936,7 +2012,19 @@ function formatEvaluationSummary(summary) {
     }
     lines.push("");
   }
+  const overallPassed = summary.passedCount === summary.total - summary.executionErrorCount || summary.qualityFailureCount === 0 && summary.executionErrorCount === 0;
+  const overallVerdict = overallPassed ? "PASS" : "FAIL";
+  const useColor = !(process.env.NO_COLOR !== void 0) && (process.stdout.isTTY ?? false);
+  const verdictColor = overallPassed ? "\x1B[32m" : "\x1B[31m";
+  const verdictText = `RESULT: ${overallVerdict}  (${summary.passedCount}/${summary.total} passed, mean score: ${formatScore(summary.mean)})`;
   lines.push("\n==================================================");
+  if (useColor) {
+    lines.push(`\x1B[1m${verdictColor}${verdictText}\x1B[0m`);
+  } else {
+    lines.push(verdictText);
+  }
+  lines.push("==================================================");
+  lines.push("");
   lines.push("EVALUATION SUMMARY");
   lines.push("==================================================");
   lines.push(`Total tests: ${summary.total}`);
@@ -3292,9 +3380,9 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
 }
 
 // src/commands/eval/targets.ts
-var ANSI_YELLOW2 = "\x1B[33m";
-var ANSI_RED2 = "\x1B[31m";
-var ANSI_RESET2 = "\x1B[0m";
+var ANSI_YELLOW3 = "\x1B[33m";
+var ANSI_RED3 = "\x1B[31m";
+var ANSI_RESET3 = "\x1B[0m";
 function isTTY() {
   return process.stdout.isTTY ?? false;
 }
@@ -3334,14 +3422,14 @@ async function selectTarget(options) {
   });
   const validationResult = await validateTargetsFile(targetsFilePath);
   const warnings = validationResult.errors.filter((e) => e.severity === "warning");
-  const useColors = isTTY();
+  const useColors2 = isTTY();
   if (warnings.length > 0) {
     console.warn(`
 Warnings in ${targetsFilePath}:`);
     for (const warning of warnings) {
       const location = warning.location ? ` [${warning.location}]` : "";
-      const prefix = useColors ? `${ANSI_YELLOW2}  \u26A0${ANSI_RESET2}` : "  \u26A0";
-      const message = useColors ? `${ANSI_YELLOW2}${warning.message}${ANSI_RESET2}` : warning.message;
+      const prefix = useColors2 ? `${ANSI_YELLOW3}  \u26A0${ANSI_RESET3}` : "  \u26A0";
+      const message = useColors2 ? `${ANSI_YELLOW3}${warning.message}${ANSI_RESET3}` : warning.message;
       console.warn(`${prefix}${location} ${message}`);
     }
     console.warn("");
@@ -3352,8 +3440,8 @@ Warnings in ${targetsFilePath}:`);
 Errors in ${targetsFilePath}:`);
     for (const error of errors) {
       const location = error.location ? ` [${error.location}]` : "";
-      const prefix = useColors ? `${ANSI_RED2}  \u2717${ANSI_RESET2}` : "  \u2717";
-      const message = useColors ? `${ANSI_RED2}${error.message}${ANSI_RESET2}` : error.message;
+      const prefix = useColors2 ? `${ANSI_RED3}  \u2717${ANSI_RESET3}` : "  \u2717";
+      const message = useColors2 ? `${ANSI_RED3}${error.message}${ANSI_RESET3}` : error.message;
       console.error(`${prefix}${location} ${message}`);
     }
     throw new Error(`Targets file validation failed with ${errors.length} error(s)`);
@@ -3425,14 +3513,14 @@ async function selectMultipleTargets(options) {
   });
   const validationResult = await validateTargetsFile(targetsFilePath);
   const warnings = validationResult.errors.filter((e) => e.severity === "warning");
-  const useColors = isTTY();
+  const useColors2 = isTTY();
   if (warnings.length > 0) {
     console.warn(`
 Warnings in ${targetsFilePath}:`);
     for (const warning of warnings) {
       const location = warning.location ? ` [${warning.location}]` : "";
-      const prefix = useColors ? `${ANSI_YELLOW2}  \u26A0${ANSI_RESET2}` : "  \u26A0";
-      const message = useColors ? `${ANSI_YELLOW2}${warning.message}${ANSI_RESET2}` : warning.message;
+      const prefix = useColors2 ? `${ANSI_YELLOW3}  \u26A0${ANSI_RESET3}` : "  \u26A0";
+      const message = useColors2 ? `${ANSI_YELLOW3}${warning.message}${ANSI_RESET3}` : warning.message;
       console.warn(`${prefix}${location} ${message}`);
     }
     console.warn("");
@@ -3443,8 +3531,8 @@ Warnings in ${targetsFilePath}:`);
 Errors in ${targetsFilePath}:`);
     for (const error of errors) {
       const location = error.location ? ` [${error.location}]` : "";
-      const prefix = useColors ? `${ANSI_RED2}  \u2717${ANSI_RESET2}` : "  \u2717";
-      const message = useColors ? `${ANSI_RED2}${error.message}${ANSI_RESET2}` : error.message;
+      const prefix = useColors2 ? `${ANSI_RED3}  \u2717${ANSI_RESET3}` : "  \u2717";
+      const message = useColors2 ? `${ANSI_RED3}${error.message}${ANSI_RESET3}` : error.message;
       console.error(`${prefix}${location} ${message}`);
     }
     throw new Error(`Targets file validation failed with ${errors.length} error(s)`);
@@ -3737,13 +3825,10 @@ async function prepareFileMetadata(params) {
       env: process.env,
       targetNames
     });
-    selections = multiSelections.map((sel) => {
-      const providerLabel = options.dryRun ? `${sel.resolvedTarget.kind} (dry-run)` : sel.resolvedTarget.kind;
-      return {
-        selection: sel,
-        inlineTargetLabel: `${sel.targetName} ${buildTargetLabelSuffix(providerLabel, sel.resolvedTarget)}`
-      };
-    });
+    selections = multiSelections.map((sel) => ({
+      selection: sel,
+      inlineTargetLabel: sel.targetName
+    }));
   } else {
     const selection = await selectTarget({
       testFilePath,
@@ -3757,11 +3842,10 @@ async function prepareFileMetadata(params) {
       dryRunDelayMax: options.dryRunDelayMax,
       env: process.env
     });
-    const providerLabel = options.dryRun ? `${selection.resolvedTarget.kind} (dry-run)` : selection.resolvedTarget.kind;
     selections = [
       {
         selection,
-        inlineTargetLabel: `${selection.targetName} ${buildTargetLabelSuffix(providerLabel, selection.resolvedTarget)}`
+        inlineTargetLabel: selection.targetName
       }
     ];
   }
@@ -3902,6 +3986,10 @@ async function runSingleEvalFile(params) {
       if (event.status === "running" && streamingObserver) {
         streamingObserver.startEvalCase(event.testId, targetName, testFilePath);
       }
+      let verdict;
+      if (event.executionStatus === "ok") verdict = "PASS";
+      else if (event.executionStatus === "quality_failure") verdict = "FAIL";
+      else if (event.executionStatus === "execution_error") verdict = "ERROR";
       progressReporter.update(displayId, {
         workerId: displayId,
         testId: matrixMode ? `${event.testId}@${targetName}` : event.testId,
@@ -3909,7 +3997,9 @@ async function runSingleEvalFile(params) {
         startedAt: event.startedAt,
         completedAt: event.completedAt,
         error: event.error,
-        targetLabel: inlineTargetLabel
+        targetLabel: inlineTargetLabel,
+        score: event.score,
+        verdict
       });
     }
   });
@@ -3973,7 +4063,7 @@ async function runEvalCommand(input) {
   const useFileExport = !!(options.otelFile || options.traceFile);
   if (options.exportOtel || useFileExport) {
     try {
-      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-OC53WD3P.js");
+      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-AFDYFH6Y.js");
       let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
       let headers = {};
       if (options.otelBackend) {
@@ -4248,6 +4338,22 @@ Results written to: ${outputPath}`);
         }
       }
     }
+    if (summary.executionErrorCount > 0 && !options.retryErrors) {
+      const evalFileArgs = resolvedTestFiles.map((f) => path12.relative(cwd, f)).join(" ");
+      const targetFlag = options.target ? ` --target ${options.target}` : "";
+      const relativeOutputPath = path12.relative(cwd, outputPath);
+      console.log(
+        `
+Tip: ${summary.executionErrorCount} execution error(s) detected. Re-run failed tests with:
+  agentv eval run ${evalFileArgs}${targetFlag} --retry-errors ${relativeOutputPath} -o ${relativeOutputPath}`
+      );
+    }
+    return {
+      executionErrorCount: summary.executionErrorCount,
+      outputPath,
+      testFiles: resolvedTestFiles,
+      target: options.target
+    };
   } finally {
     unsubscribeCodexLogs();
     unsubscribePiLogs();
@@ -4285,6 +4391,10 @@ export {
   HtmlWriter,
   resolveEvalPaths,
   findRepoRoot,
+  buildGradingArtifact,
+  buildTimingArtifact,
+  buildBenchmarkArtifact,
+  parseJsonlResults,
   detectFileType,
   validateEvalFile,
   validateTargetsFile,
@@ -4295,4 +4405,4 @@ export {
   selectTarget,
   runEvalCommand
 };
-//# sourceMappingURL=chunk-4ZMSAQWS.js.map
+//# sourceMappingURL=chunk-RE5I3U2S.js.map