agentv 3.3.0 → 3.5.0

package/dist/{chunk-4ZMSAQWS.js → chunk-RLL4QGNL.js}

@@ -25,12 +25,12 @@ import {
   subscribeToCopilotCliLogEntries,
   subscribeToCopilotSdkLogEntries,
   subscribeToPiLogEntries
-} from "./chunk-5M3K2DMV.js";
+} from "./chunk-D6G4N2H2.js";
 
 // package.json
 var package_default = {
   name: "agentv",
-  version: "3.3.0",
+  version: "3.5.0",
   description: "CLI entry point for AgentV",
   type: "module",
   repository: {
@@ -320,34 +320,12 @@ function parseWorkspaceChanges(fileChanges) {
     diff_summary: diffSummary
   };
 }
-function buildExpectations(result) {
-  const expectations = [];
-  if (result.scores && result.scores.length > 0) {
-    for (const evaluator of result.scores) {
-      for (const hit of evaluator.hits) {
-        expectations.push({
-          text: hit,
-          passed: true,
-          evidence: evaluator.reasoning ?? ""
-        });
-      }
-      for (const miss of evaluator.misses) {
-        expectations.push({
-          text: miss,
-          passed: false,
-          evidence: evaluator.reasoning ?? ""
-        });
-      }
-    }
-  } else {
-    for (const hit of result.hits) {
-      expectations.push({ text: hit, passed: true, evidence: result.reasoning ?? "" });
-    }
-    for (const miss of result.misses) {
-      expectations.push({ text: miss, passed: false, evidence: result.reasoning ?? "" });
-    }
-  }
-  return expectations;
+function buildAssertions(result) {
+  return result.assertions.map((a) => ({
+    text: a.text,
+    passed: a.passed,
+    evidence: a.evidence ?? ""
+  }));
 }
 function buildEvaluators(scores) {
   if (!scores || scores.length === 0) {
@@ -357,23 +335,22 @@ function buildEvaluators(scores) {
     name: s.name,
     type: s.type,
     score: s.score,
-    reasoning: s.reasoning ?? "",
+    reasoning: "",
     weight: s.weight,
     verdict: s.verdict,
-    hits: s.hits,
-    misses: s.misses,
+    assertions: s.assertions,
     details: s.details
   }));
 }
 function buildGradingArtifact(result) {
-  const expectations = buildExpectations(result);
-  const passed = expectations.filter((e) => e.passed).length;
-  const failed = expectations.filter((e) => !e.passed).length;
-  const total = expectations.length;
+  const assertions = buildAssertions(result);
+  const passed = assertions.filter((e) => e.passed).length;
+  const failed = assertions.filter((e) => !e.passed).length;
+  const total = assertions.length;
   const { toolCalls, total: totalToolCalls } = countToolCalls(result);
   const errorsEncountered = result.error ? 1 : 0;
   return {
-    expectations,
+    assertions,
     summary: {
       passed,
       failed,
@@ -496,6 +473,42 @@ function buildBenchmarkArtifact(results, evalFile = "") {
     notes
   };
 }
+function toCamelCase(str) {
+  return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
+}
+function toCamelCaseDeep(obj) {
+  if (obj === null || obj === void 0) {
+    return obj;
+  }
+  if (Array.isArray(obj)) {
+    return obj.map((item) => toCamelCaseDeep(item));
+  }
+  if (typeof obj === "object") {
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[toCamelCase(key)] = toCamelCaseDeep(value);
+    }
+    return result;
+  }
+  return obj;
+}
+function parseJsonlResults(content) {
+  const results = [];
+  const lines = content.split("\n");
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed.length === 0) {
+      continue;
+    }
+    try {
+      const parsed = JSON.parse(trimmed);
+      const camelCased = toCamelCaseDeep(parsed);
+      results.push(camelCased);
+    } catch {
+    }
+  }
+  return results;
+}
 async function writeArtifactsFromResults(results, outputDir, options) {
   const gradingDir = path3.join(outputDir, "grading");
   const timingPath = path3.join(outputDir, "timing.json");
@@ -1321,23 +1334,27 @@ var SCRIPT = `
     /* evaluator results */
     if(r.scores&&r.scores.length>0){
       h+="<h4>Evaluator Results</h4>";
-      h+='<table class="eval-table"><thead><tr><th>Evaluator</th><th>Score</th><th>Status</th><th>Reasoning</th></tr></thead><tbody>';
+      h+='<table class="eval-table"><thead><tr><th>Evaluator</th><th>Score</th><th>Status</th><th>Assertions</th></tr></thead><tbody>';
       for(var i=0;i<r.scores.length;i++){
         var ev=r.scores[i],evS=ev.score>=0.5?"pass":"fail";
-        h+="<tr><td class=\\"fw-medium\\">"+esc(ev.name)+'</td><td class="'+sCls(ev.score)+'">'+fmtPct(ev.score)+"</td><td>"+sIcon(evS)+'</td><td class="reasoning-cell">'+esc(ev.reasoning||"")+"</td></tr>";
+        var evAssertions=ev.assertions||[];
+        var evSummary=evAssertions.map(function(a){return (a.passed?"\u2713 ":"\u2717 ")+a.text;}).join("; ");
+        h+="<tr><td class=\\"fw-medium\\">"+esc(ev.name)+'</td><td class="'+sCls(ev.score)+'">'+fmtPct(ev.score)+"</td><td>"+sIcon(evS)+'</td><td class="reasoning-cell">'+esc(evSummary)+"</td></tr>";
       }
       h+="</tbody></table>";
     }
 
-    /* hits / misses */
-    if(r.hits&&r.hits.length>0){
-      h+='<h4>Passed Expectations</h4><ul class="expect-list pass">';
-      for(var i=0;i<r.hits.length;i++)h+="<li>"+esc(r.hits[i])+"</li>";
+    /* assertions */
+    var passedA=r.assertions?r.assertions.filter(function(a){return a.passed;}):[];
+    var failedA=r.assertions?r.assertions.filter(function(a){return !a.passed;}):[];
+    if(passedA.length>0){
+      h+='<h4>Passed Assertions</h4><ul class="expect-list pass">';
+      for(var i=0;i<passedA.length;i++)h+="<li>"+esc(passedA[i].text)+(passedA[i].evidence?" <span class=\\"reasoning-cell\\">("+esc(passedA[i].evidence)+")</span>":"")+"</li>";
       h+="</ul>";
     }
-    if(r.misses&&r.misses.length>0){
-      h+='<h4>Failed Expectations</h4><ul class="expect-list fail">';
-      for(var i=0;i<r.misses.length;i++)h+="<li>"+esc(r.misses[i])+"</li>";
+    if(failedA.length>0){
+      h+='<h4>Failed Assertions</h4><ul class="expect-list fail">';
+      for(var i=0;i<failedA.length;i++)h+="<li>"+esc(failedA[i].text)+(failedA[i].evidence?" <span class=\\"reasoning-cell\\">("+esc(failedA[i].evidence)+")</span>":"")+"</li>";
       h+="</ul>";
     }
 
@@ -1526,10 +1543,10 @@ var JunitWriter = class _JunitWriter {
     `;
         } else if (r.score < 0.5) {
           const message = `score=${r.score.toFixed(3)}`;
+          const failedAssertions = r.assertions.filter((a) => !a.passed);
           const detail = [
             `Score: ${r.score.toFixed(3)}`,
-            r.reasoning ? `Reasoning: ${r.reasoning}` : "",
-            r.misses.length > 0 ? `Misses: ${r.misses.join(", ")}` : ""
+            failedAssertions.length > 0 ? `Failed: ${failedAssertions.map((a) => a.text).join(", ")}` : ""
           ].filter(Boolean).join("\n");
           inner = `
       <failure message="${escapeXml(message)}">${escapeXml(detail)}</failure>
@@ -1673,6 +1690,24 @@ async function createMultiWriter(filePaths) {
 }
 
 // src/commands/eval/progress-display.ts
+var ANSI_BOLD = "\x1B[1m";
+var ANSI_GREEN = "\x1B[32m";
+var ANSI_RED2 = "\x1B[31m";
+var ANSI_YELLOW2 = "\x1B[33m";
+var ANSI_RESET2 = "\x1B[0m";
+function useColors() {
+  if (process.env.NO_COLOR !== void 0) return false;
+  return process.stdout.isTTY ?? false;
+}
+function formatVerdict(score, verdict) {
+  if (verdict === void 0) return "";
+  const colors = useColors();
+  const scoreStr = score !== void 0 ? score.toFixed(3) : "";
+  const verdictLabel = verdict === "ERROR" ? "ERROR" : `${scoreStr} ${verdict}`;
+  if (!colors) return ` | ${verdictLabel}`;
+  const color = verdict === "PASS" ? ANSI_GREEN : verdict === "FAIL" ? ANSI_RED2 : ANSI_YELLOW2;
+  return ` | ${color}${ANSI_BOLD}${verdictLabel}${ANSI_RESET2}`;
+}
 var ProgressDisplay = class {
   workers = /* @__PURE__ */ new Map();
   totalTests = 0;
@@ -1716,11 +1751,13 @@ var ProgressDisplay = class {
         }
         break;
       case "completed":
-        console.log(`${countPrefix}   \u2705 ${progress.testId}${targetSuffix}`);
+        console.log(
+          `${countPrefix}   \u2705 ${progress.testId}${targetSuffix}${formatVerdict(progress.score, progress.verdict)}`
+        );
         break;
       case "failed":
         console.log(
-          `${countPrefix}   \u274C ${progress.testId}${targetSuffix}${progress.error ? `: ${progress.error}` : ""}`
+          `${countPrefix}   \u274C ${progress.testId}${targetSuffix}${formatVerdict(progress.score, progress.verdict)}${progress.error ? `: ${progress.error}` : ""}`
         );
         break;
     }
@@ -1760,6 +1797,22 @@ var ProgressDisplay = class {
 // src/commands/eval/retry-errors.ts
 import { createReadStream } from "node:fs";
 import { createInterface } from "node:readline";
+function getTestId(result) {
+  return result.testId ?? result.test_id;
+}
+function getExecutionStatus(result) {
+  return result.executionStatus ?? result.execution_status;
+}
+function toEvaluationResult(result) {
+  if (result.testId !== void 0 && result.executionStatus !== void 0) {
+    return result;
+  }
+  return {
+    ...result,
+    testId: getTestId(result) ?? "",
+    executionStatus: getExecutionStatus(result)
+  };
+}
 async function loadErrorTestIds(jsonlPath) {
   const ids = [];
   const rl = createInterface({
@@ -1771,8 +1824,10 @@ async function loadErrorTestIds(jsonlPath) {
     if (!trimmed) continue;
     try {
       const parsed = JSON.parse(trimmed);
-      if (parsed.executionStatus === "execution_error" && parsed.testId) {
-        ids.push(parsed.testId);
+      const executionStatus = getExecutionStatus(parsed);
+      const testId = getTestId(parsed);
+      if (executionStatus === "execution_error" && testId) {
+        ids.push(testId);
       }
     } catch {
     }
@@ -1790,9 +1845,11 @@ async function loadNonErrorResults(jsonlPath) {
     if (!trimmed) continue;
     try {
       const parsed = JSON.parse(trimmed);
-      if (!parsed.testId || parsed.score === void 0) continue;
-      if (parsed.executionStatus !== "execution_error") {
-        results.push(parsed);
+      const testId = getTestId(parsed);
+      const executionStatus = getExecutionStatus(parsed);
+      if (!testId || parsed.score === void 0) continue;
+      if (executionStatus !== "execution_error") {
+        results.push(toEvaluationResult(parsed));
       }
     } catch {
     }
@@ -1936,7 +1993,19 @@ function formatEvaluationSummary(summary) {
     }
     lines.push("");
   }
+  const overallPassed = summary.passedCount === summary.total - summary.executionErrorCount || summary.qualityFailureCount === 0 && summary.executionErrorCount === 0;
+  const overallVerdict = overallPassed ? "PASS" : "FAIL";
+  const useColor = !(process.env.NO_COLOR !== void 0) && (process.stdout.isTTY ?? false);
+  const verdictColor = overallPassed ? "\x1B[32m" : "\x1B[31m";
+  const verdictText = `RESULT: ${overallVerdict}  (${summary.passedCount}/${summary.total} passed, mean score: ${formatScore(summary.mean)})`;
   lines.push("\n==================================================");
+  if (useColor) {
+    lines.push(`\x1B[1m${verdictColor}${verdictText}\x1B[0m`);
+  } else {
+    lines.push(verdictText);
+  }
+  lines.push("==================================================");
+  lines.push("");
   lines.push("EVALUATION SUMMARY");
   lines.push("==================================================");
   lines.push(`Total tests: ${summary.total}`);
@@ -3292,9 +3361,9 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
 }
 
 // src/commands/eval/targets.ts
-var ANSI_YELLOW2 = "\x1B[33m";
-var ANSI_RED2 = "\x1B[31m";
-var ANSI_RESET2 = "\x1B[0m";
+var ANSI_YELLOW3 = "\x1B[33m";
+var ANSI_RED3 = "\x1B[31m";
+var ANSI_RESET3 = "\x1B[0m";
 function isTTY() {
   return process.stdout.isTTY ?? false;
 }
@@ -3334,14 +3403,14 @@ async function selectTarget(options) {
   });
   const validationResult = await validateTargetsFile(targetsFilePath);
   const warnings = validationResult.errors.filter((e) => e.severity === "warning");
-  const useColors = isTTY();
+  const useColors2 = isTTY();
   if (warnings.length > 0) {
     console.warn(`
 Warnings in ${targetsFilePath}:`);
     for (const warning of warnings) {
       const location = warning.location ? ` [${warning.location}]` : "";
-      const prefix = useColors ? `${ANSI_YELLOW2}  \u26A0${ANSI_RESET2}` : "  \u26A0";
-      const message = useColors ? `${ANSI_YELLOW2}${warning.message}${ANSI_RESET2}` : warning.message;
+      const prefix = useColors2 ? `${ANSI_YELLOW3}  \u26A0${ANSI_RESET3}` : "  \u26A0";
+      const message = useColors2 ? `${ANSI_YELLOW3}${warning.message}${ANSI_RESET3}` : warning.message;
       console.warn(`${prefix}${location} ${message}`);
     }
     console.warn("");
@@ -3352,8 +3421,8 @@ Warnings in ${targetsFilePath}:`);
 Errors in ${targetsFilePath}:`);
     for (const error of errors) {
       const location = error.location ? ` [${error.location}]` : "";
-      const prefix = useColors ? `${ANSI_RED2}  \u2717${ANSI_RESET2}` : "  \u2717";
-      const message = useColors ? `${ANSI_RED2}${error.message}${ANSI_RESET2}` : error.message;
+      const prefix = useColors2 ? `${ANSI_RED3}  \u2717${ANSI_RESET3}` : "  \u2717";
+      const message = useColors2 ? `${ANSI_RED3}${error.message}${ANSI_RESET3}` : error.message;
       console.error(`${prefix}${location} ${message}`);
     }
     throw new Error(`Targets file validation failed with ${errors.length} error(s)`);
@@ -3425,14 +3494,14 @@ async function selectMultipleTargets(options) {
   });
   const validationResult = await validateTargetsFile(targetsFilePath);
   const warnings = validationResult.errors.filter((e) => e.severity === "warning");
-  const useColors = isTTY();
+  const useColors2 = isTTY();
   if (warnings.length > 0) {
     console.warn(`
 Warnings in ${targetsFilePath}:`);
     for (const warning of warnings) {
       const location = warning.location ? ` [${warning.location}]` : "";
-      const prefix = useColors ? `${ANSI_YELLOW2}  \u26A0${ANSI_RESET2}` : "  \u26A0";
-      const message = useColors ? `${ANSI_YELLOW2}${warning.message}${ANSI_RESET2}` : warning.message;
+      const prefix = useColors2 ? `${ANSI_YELLOW3}  \u26A0${ANSI_RESET3}` : "  \u26A0";
+      const message = useColors2 ? `${ANSI_YELLOW3}${warning.message}${ANSI_RESET3}` : warning.message;
       console.warn(`${prefix}${location} ${message}`);
     }
     console.warn("");
@@ -3443,8 +3512,8 @@ Warnings in ${targetsFilePath}:`);
 Errors in ${targetsFilePath}:`);
     for (const error of errors) {
       const location = error.location ? ` [${error.location}]` : "";
-      const prefix = useColors ? `${ANSI_RED2}  \u2717${ANSI_RESET2}` : "  \u2717";
-      const message = useColors ? `${ANSI_RED2}${error.message}${ANSI_RESET2}` : error.message;
+      const prefix = useColors2 ? `${ANSI_RED3}  \u2717${ANSI_RESET3}` : "  \u2717";
+      const message = useColors2 ? `${ANSI_RED3}${error.message}${ANSI_RESET3}` : error.message;
       console.error(`${prefix}${location} ${message}`);
     }
     throw new Error(`Targets file validation failed with ${errors.length} error(s)`);
@@ -3737,13 +3806,10 @@ async function prepareFileMetadata(params) {
       env: process.env,
       targetNames
     });
-    selections = multiSelections.map((sel) => {
-      const providerLabel = options.dryRun ? `${sel.resolvedTarget.kind} (dry-run)` : sel.resolvedTarget.kind;
-      return {
-        selection: sel,
-        inlineTargetLabel: `${sel.targetName} ${buildTargetLabelSuffix(providerLabel, sel.resolvedTarget)}`
-      };
-    });
+    selections = multiSelections.map((sel) => ({
+      selection: sel,
+      inlineTargetLabel: sel.targetName
+    }));
   } else {
     const selection = await selectTarget({
       testFilePath,
@@ -3757,11 +3823,10 @@ async function prepareFileMetadata(params) {
       dryRunDelayMax: options.dryRunDelayMax,
       env: process.env
     });
-    const providerLabel = options.dryRun ? `${selection.resolvedTarget.kind} (dry-run)` : selection.resolvedTarget.kind;
     selections = [
       {
         selection,
-        inlineTargetLabel: `${selection.targetName} ${buildTargetLabelSuffix(providerLabel, selection.resolvedTarget)}`
+        inlineTargetLabel: selection.targetName
       }
     ];
   }
@@ -3902,6 +3967,10 @@ async function runSingleEvalFile(params) {
       if (event.status === "running" && streamingObserver) {
         streamingObserver.startEvalCase(event.testId, targetName, testFilePath);
       }
+      let verdict;
+      if (event.executionStatus === "ok") verdict = "PASS";
+      else if (event.executionStatus === "quality_failure") verdict = "FAIL";
+      else if (event.executionStatus === "execution_error") verdict = "ERROR";
       progressReporter.update(displayId, {
         workerId: displayId,
         testId: matrixMode ? `${event.testId}@${targetName}` : event.testId,
@@ -3909,7 +3978,9 @@ async function runSingleEvalFile(params) {
         startedAt: event.startedAt,
         completedAt: event.completedAt,
         error: event.error,
-        targetLabel: inlineTargetLabel
+        targetLabel: inlineTargetLabel,
+        score: event.score,
+        verdict
       });
     }
   });
@@ -3973,7 +4044,7 @@ async function runEvalCommand(input) {
   const useFileExport = !!(options.otelFile || options.traceFile);
   if (options.exportOtel || useFileExport) {
     try {
-      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-OC53WD3P.js");
+      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-MZFXE6B5.js");
       let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
       let headers = {};
       if (options.otelBackend) {
@@ -4248,6 +4319,22 @@ Results written to: ${outputPath}`);
         }
       }
     }
+    if (summary.executionErrorCount > 0 && !options.retryErrors) {
+      const evalFileArgs = resolvedTestFiles.map((f) => path12.relative(cwd, f)).join(" ");
+      const targetFlag = options.target ? ` --target ${options.target}` : "";
+      const relativeOutputPath = path12.relative(cwd, outputPath);
+      console.log(
+        `
+Tip: ${summary.executionErrorCount} execution error(s) detected. Re-run failed tests with:
+  agentv eval run ${evalFileArgs}${targetFlag} --retry-errors ${relativeOutputPath} -o ${relativeOutputPath}`
+      );
+    }
+    return {
+      executionErrorCount: summary.executionErrorCount,
+      outputPath,
+      testFiles: resolvedTestFiles,
+      target: options.target
+    };
   } finally {
     unsubscribeCodexLogs();
     unsubscribePiLogs();
@@ -4285,6 +4372,10 @@ export {
   HtmlWriter,
   resolveEvalPaths,
   findRepoRoot,
+  buildGradingArtifact,
+  buildTimingArtifact,
+  buildBenchmarkArtifact,
+  parseJsonlResults,
   detectFileType,
   validateEvalFile,
   validateTargetsFile,
@@ -4295,4 +4386,4 @@ export {
   selectTarget,
   runEvalCommand
 };
-//# sourceMappingURL=chunk-4ZMSAQWS.js.map
+//# sourceMappingURL=chunk-RLL4QGNL.js.map