@vitest-evals/github-reporter 0.16.1 → 0.17.0

package/dist/cli.js

@@ -323,7 +323,8 @@ function collectEvalReport(input, options = {}) {
   const cases = workspace.cases.map(collectEvalCase);
   const failures = cases.filter((testCase) => testCase.status === "failed");
   const evalScores = cases.map((testCase) => testCase.eval?.avgScore).filter((score) => isFiniteNumber(score));
-  const usage2 = sumUsage(cases);
+  const usage2 = sumAppUsage(cases);
+  const judgeUsage = sumJudgeUsage(cases);
   const durationMs = workspace.runs[0]?.durationMs;
   return {
     status: input.success && failures.length === 0 ? "passed" : "failed",
@@ -343,6 +344,7 @@ function collectEvalReport(input, options = {}) {
       minimum: Math.min(...evalScores)
     } : void 0,
     usage: usage2,
+    judgeUsage,
     cases,
     failures
   };
@@ -467,24 +469,67 @@ function stringifyReason(value) {
   }
   return typeof value === "string" ? value : stringifyValue(value, 4e3);
 }
-function sumUsage(cases) {
-  const usage2 = {
+function emptyUsage() {
+  return {
     inputTokens: 0,
     outputTokens: 0,
     reasoningTokens: 0,
     totalTokens: 0,
     toolCalls: 0
   };
+}
+function addRunUsage(total, usage2) {
+  total.inputTokens += usage2?.inputTokens ?? 0;
+  total.outputTokens += usage2?.outputTokens ?? 0;
+  total.reasoningTokens += usage2?.reasoningTokens ?? 0;
+  total.totalTokens += usage2?.totalTokens ?? (usage2?.inputTokens ?? 0) + (usage2?.outputTokens ?? 0) + (usage2?.reasoningTokens ?? 0);
+  if (usage2?.costUsd !== void 0) {
+    total.costUsd = (total.costUsd ?? 0) + usage2.costUsd;
+  }
+  total.toolCalls += usage2?.toolCalls ?? 0;
+}
+function sumAppUsage(cases) {
+  const usage2 = emptyUsage();
+  const runUsages = cases.map(appUsageForCase).filter((item) => item !== void 0);
+  for (const runUsage of runUsages) {
+    addRunUsage(usage2, runUsage);
+  }
+  omitPartialCost(usage2, runUsages);
+  return usage2;
+}
+function appUsageForCase(testCase) {
+  const usage2 = testCase.harness?.usage;
+  const effectiveToolCalls = toolCallCount(testCase);
+  if (!usage2 && effectiveToolCalls === 0) {
+    return void 0;
+  }
+  return {
+    ...usage2,
+    ...effectiveToolCalls > 0 ? { toolCalls: effectiveToolCalls } : {}
+  };
+}
+function sumJudgeUsage(cases) {
+  const usage2 = emptyUsage();
+  const runUsages = [];
   for (const testCase of cases) {
-    const caseUsage = testCase.harness?.usage;
-    usage2.inputTokens += caseUsage?.inputTokens ?? 0;
-    usage2.outputTokens += caseUsage?.outputTokens ?? 0;
-    usage2.reasoningTokens += caseUsage?.reasoningTokens ?? 0;
-    usage2.totalTokens += caseUsage?.totalTokens ?? (caseUsage?.inputTokens ?? 0) + (caseUsage?.outputTokens ?? 0) + (caseUsage?.reasoningTokens ?? 0);
-    usage2.toolCalls += toolCallCount(testCase);
+    for (const score of testCase.eval?.scores ?? []) {
+      for (const run of score.judgeRuns ?? []) {
+        runUsages.push(run.usage);
+        addRunUsage(usage2, run.usage);
+      }
+    }
   }
+  omitPartialCost(usage2, runUsages);
   return usage2;
 }
+function omitPartialCost(total, usages) {
+  if (usages.some(hasUsageWithoutCost)) {
+    total.costUsd = void 0;
+  }
+}
+function hasUsageWithoutCost(usage2) {
+  return usage2?.costUsd === void 0 && ((usage2?.totalTokens ?? (usage2?.inputTokens ?? 0) + (usage2?.outputTokens ?? 0) + (usage2?.reasoningTokens ?? 0)) > 0 || (usage2?.toolCalls ?? 0) > 0);
+}
 function toolCallCount(testCase) {
   const usageToolCalls = testCase.harness?.usage?.toolCalls;
   if (usageToolCalls !== void 0) {
@@ -521,8 +566,8 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate,
-      title: "Eval report hard failure",
-      message: `${formatNumber(nonEvalFailures)} non-eval test failure${nonEvalFailures === 1 ? "" : "s"}; ${counts}`
+      title: "Eval run failed",
+      message: `${formatNumber(nonEvalFailures)} test failure${nonEvalFailures === 1 ? "" : "s"} outside eval cases; ${counts}`
     };
   }
   if (report.totals.evalTotal === 0) {
@@ -531,7 +576,7 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate: null,
-      title: "Eval report hard failure",
+      title: "Eval run failed",
       message: "no eval cases were reported"
     };
   }
@@ -541,8 +586,8 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate,
-      title: "Eval report hard failure",
-      message: `vitest run failed without counted test failures; ${counts}`
+      title: "Eval run failed",
+      message: `Vitest failed without reporting a failed test; ${counts}`
     };
   }
   if (minPassRate !== void 0 && (passRate === null || passRate + Number.EPSILON < minPassRate)) {
@@ -552,7 +597,7 @@ function evaluateEvalGate(report, policy = {}) {
       enforced: true,
       passRate,
       title: `Eval pass rate ${formatPercent(passRate)} \u2014 required ${formatPercent(minPassRate)}`,
-      message: `eval pass rate below floor: ${counts}; required >= ${formatPercent(minPassRate)}`
+      message: `pass rate is below the minimum: ${counts}; minimum ${formatPercent(minPassRate)}`
     };
   }
   if (minScoreAverage !== void 0) {
@@ -563,8 +608,8 @@ function evaluateEvalGate(report, policy = {}) {
         status: "failed",
         enforced: true,
         passRate,
-        title: "Eval score gate failed",
-        message: `no score average available; required avg score >= ${formatScore(minScoreAverage)}`
+        title: "Average score unavailable",
+        message: `no average score was reported; minimum ${formatScore(minScoreAverage)}`
       };
     }
     if (average + Number.EPSILON < minScoreAverage) {
@@ -573,8 +618,8 @@ function evaluateEvalGate(report, policy = {}) {
         status: "failed",
         enforced: true,
         passRate,
-        title: `Avg score ${formatScore(average)} \u2014 required ${formatScore(minScoreAverage)}`,
-        message: `avg score below floor: ${counts}; required avg score >= ${formatScore(minScoreAverage)}`
+        title: `Average score ${formatScore(average)} \u2014 required ${formatScore(minScoreAverage)}`,
+        message: `average score is below the minimum: ${counts}; minimum ${formatScore(minScoreAverage)}`
       };
     }
   }
@@ -584,7 +629,7 @@ function evaluateEvalGate(report, policy = {}) {
     enforced: true,
     passRate,
     title: enforcedPassTitle(report, passRate, minPassRate, minScoreAverage),
-    message: `eval gate passed: ${counts}${formatFloorSuffix(minPassRate, minScoreAverage)}`
+    message: `requirements met: ${counts}${formatMinimumSuffix(minPassRate, minScoreAverage)}`
   };
 }
 function computePassRate(report) {
@@ -617,7 +662,7 @@ function resolveMinPassRate(policy) {
 }
 function defaultCheckTitle(report) {
   if (report.failures.length === 0 && report.status === "passed") {
-    return "No eval failures";
+    return "No eval cases failed";
   }
   if (report.failures.length === 0) {
     return "Vitest run failed";
@@ -626,10 +671,10 @@ function defaultCheckTitle(report) {
 }
 function enforcedPassTitle(report, passRate, minPassRate, minScoreAverage) {
   if (minPassRate !== void 0) {
-    return `Eval pass rate ${formatPercent(passRate)} \u2014 floor ${formatPercent(minPassRate)}`;
+    return `Eval pass rate ${formatPercent(passRate)} \u2014 minimum ${formatPercent(minPassRate)}`;
   }
   if (minScoreAverage !== void 0) {
-    return `Avg score ${formatScore(report.score?.average)} \u2014 floor ${formatScore(minScoreAverage)}`;
+    return `Average score ${formatScore(report.score?.average)} \u2014 minimum ${formatScore(minScoreAverage)}`;
   }
   return defaultCheckTitle(report);
 }
@@ -638,15 +683,15 @@ function formatEvalCounts(report, passRate) {
   const passRateText = passRate === null ? "n/a" : formatPercent(passRate);
   return `${formatNumber(report.totals.evalPassed)}/${formatNumber(
     report.totals.evalTotal
-  )} passed (${passRateText}), avg score ${scoreText}`;
+  )} passed (${passRateText}), average score ${scoreText}`;
 }
-function formatFloorSuffix(minPassRate, minScoreAverage) {
+function formatMinimumSuffix(minPassRate, minScoreAverage) {
   const parts = [];
   if (minPassRate !== void 0) {
-    parts.push(`pass rate floor ${formatPercent(minPassRate)}`);
+    parts.push(`minimum pass rate ${formatPercent(minPassRate)}`);
   }
   if (minScoreAverage !== void 0) {
-    parts.push(`avg score floor ${formatScore(minScoreAverage)}`);
+    parts.push(`minimum average score ${formatScore(minScoreAverage)}`);
   }
   return parts.length === 0 ? "" : `; ${parts.join(", ")}`;
 }
@@ -681,23 +726,23 @@ function renderJobSummary(report, options = {}) {
     ""
   ];
   if (report.failures.length > 0) {
-    const failureHeading = options.gate?.ok === true ? "### Quality Misses" : "### Failures";
+    const failureHeading = options.gate?.ok === true ? "### Cases Below Target" : "### Failures";
     lines.push(failureHeading, "");
     failures.forEach((testCase, index) => {
       lines.push(...renderFailureDetails(testCase, index + 1, options), "");
     });
     if (report.failures.length > failures.length) {
-      const omittedLabel = options.gate?.ok === true ? "quality misses" : "failures";
+      const omittedLabel = options.gate?.ok === true ? "cases below target" : "failures";
       lines.push(
         `${report.failures.length - failures.length} more ${omittedLabel} omitted from this summary.`,
         ""
       );
     }
   } else if (report.totals.evalTotal > 0) {
-    lines.push("### Failures", "", "No eval failures.", "");
+    lines.push("### Failures", "", "No eval cases failed.", "");
   }
   if (report.totals.evalTotal === 0) {
-    lines.push("No eval metadata was found in the Vitest JSON report.", "");
+    lines.push("No eval results were found in the Vitest JSON report.", "");
   }
   return `${lines.join("\n")}
 `;
@@ -707,9 +752,9 @@ function formatCountLine(passed, failed, total) {
 }
 function renderSummaryTable(report, nonEvalFailures, gate) {
   const rows = [
-    ["Status", gate?.status ?? report.status],
+    ["Status", capitalize(gate?.status ?? report.status)],
     [
-      "Evals",
+      "Eval cases",
       formatCountLine(
         report.totals.evalPassed,
         report.totals.evalFailed,
@@ -718,23 +763,31 @@ function renderSummaryTable(report, nonEvalFailures, gate) {
     ]
   ];
   if (gate?.passRate !== void 0 && gate.passRate !== null) {
-    rows.push(["Pass Rate", formatPercent(gate.passRate)]);
+    rows.push(["Pass rate", formatPercent(gate.passRate)]);
   } else if (report.totals.evalTotal > 0) {
     rows.push([
-      "Pass Rate",
+      "Pass rate",
       formatPercent(report.totals.evalPassed / report.totals.evalTotal)
     ]);
   }
   if (report.score) {
     rows.push(["Score", formatScoreSummary(report.score)]);
   }
+  if (hasUsage(report.usage) || hasUsage(report.judgeUsage)) {
+    rows.push(["App usage", formatUsage(report.usage)]);
+    rows.push(["Judge usage", formatUsage(report.judgeUsage)]);
+    rows.push([
+      "Total usage",
+      formatUsage(sumUsage(report.usage, report.judgeUsage))
+    ]);
+  }
   if (gate?.enforced) {
-    rows.push(["Gate", gate.message]);
+    rows.push(["Requirements", gate.message]);
   }
   if (nonEvalFailures > 0) {
     rows.push([
-      "Other Failures",
-      `${formatNumber(nonEvalFailures)} non-eval test failure${nonEvalFailures === 1 ? "" : "s"}`
+      "Other test failures",
+      `${formatNumber(nonEvalFailures)} test failure${nonEvalFailures === 1 ? "" : "s"} outside eval cases`
     ]);
   }
   rows.push(["Duration", formatDuration(report.durationMs)]);
@@ -747,7 +800,53 @@ function renderSummaryTable(report, nonEvalFailures, gate) {
   ];
 }
 function formatScoreSummary(score) {
-  return `avg ${formatScore(score.average)}${score.minimum === void 0 ? "" : `, min ${formatScore(score.minimum)}`}`;
+  return `average ${formatScore(score.average)}${score.minimum === void 0 ? "" : `, lowest ${formatScore(score.minimum)}`}`;
+}
+function capitalize(value) {
+  return value.charAt(0).toUpperCase() + value.slice(1);
+}
+function hasUsage(usage2) {
+  return usage2.totalTokens > 0 || usage2.costUsd !== void 0 || usage2.toolCalls > 0;
+}
+function sumUsage(app, judge) {
+  const appCostKnown = !hasUsageWithoutCost2(app);
+  const judgeCostKnown = !hasUsageWithoutCost2(judge);
+  return {
+    inputTokens: app.inputTokens + judge.inputTokens,
+    outputTokens: app.outputTokens + judge.outputTokens,
+    reasoningTokens: app.reasoningTokens + judge.reasoningTokens,
+    totalTokens: app.totalTokens + judge.totalTokens,
+    ...appCostKnown && judgeCostKnown ? { costUsd: (app.costUsd ?? 0) + (judge.costUsd ?? 0) } : {},
+    toolCalls: app.toolCalls + judge.toolCalls
+  };
+}
+function hasUsageWithoutCost2(usage2) {
+  return usage2.costUsd === void 0 && (usage2.totalTokens > 0 || usage2.toolCalls > 0);
+}
+function formatUsage(usage2) {
+  const parts = [];
+  if (usage2.totalTokens > 0) {
+    parts.push(`${formatNumber(usage2.totalTokens)} tokens`);
+  }
+  if (usage2.costUsd !== void 0) {
+    parts.push(
+      usage2.costUsd.toLocaleString("en-US", {
+        style: "currency",
+        currency: "USD",
+        minimumFractionDigits: 2,
+        maximumFractionDigits: 6
+      })
+    );
+  }
+  if (hasUsageWithoutCost2(usage2)) {
+    parts.push("cost unavailable");
+  }
+  if (usage2.toolCalls > 0) {
+    parts.push(
+      `${formatNumber(usage2.toolCalls)} tool call${usage2.toolCalls === 1 ? "" : "s"}`
+    );
+  }
+  return parts.join(", ") || "none";
 }
 function escapeTableCell(value) {
   return value.replace(/\r?\n/g, " ").replace(/\\/g, "\\\\").replace(/\|/g, "\\|");
@@ -827,7 +926,7 @@ function renderFailureBlock(testCase, {
     ["Case", `${number}. ${testCase.displayName}`],
     ["Status", testCase.status],
     ["Location", formatLocation(testCase.displayFile, testCase.location)],
-    ["Harness", testCase.harness?.name ?? "n/a"],
+    ["App runner", testCase.harness?.name ?? "n/a"],
     ["Score", formatScore(failure?.score ?? testCase.eval?.avgScore)],
     ["Judge", failure?.judgeName ?? "n/a"]
   ];
@@ -865,7 +964,7 @@ function renderFailureBlock(testCase, {
   if (finalOutput !== void 0) {
     lines.push(
       ...renderAsciiSection(
-        "Final Output",
+        "Output",
         stringifyValue(finalOutput, maxOutputChars).split(/\r?\n/)
       ),
       ""
@@ -893,7 +992,7 @@ function renderFailureBlock(testCase, {
   if (testCase.harness?.errors.length) {
     lines.push(
       ...renderAsciiSection(
-        "Harness Errors",
+        "App errors",
         stringifyValue(testCase.harness.errors, maxReasonChars).split(/\r?\n/)
       ),
       ""
@@ -942,7 +1041,9 @@ function formatCaseUsage(testCase) {
     parts.push(`${formatNumber(totalTokens)} tokens`);
   }
   if (toolCalls > 0) {
-    parts.push(`${formatNumber(toolCalls)} tool${toolCalls === 1 ? "" : "s"}`);
+    parts.push(
+      `${formatNumber(toolCalls)} tool call${toolCalls === 1 ? "" : "s"}`
+    );
   }
   if (testCase.harness?.timingMs !== void 0) {
     parts.push(formatDuration(testCase.harness.timingMs));
@@ -1108,16 +1209,22 @@ function mergeEvalReports(reports) {
       minimum: Math.min(...scoredCases)
     } : void 0,
     usage: mergeUsage(reports.map((report) => report.usage)),
+    judgeUsage: mergeUsage(reports.map((report) => report.judgeUsage)),
     cases,
     failures
   };
 }
 function mergeUsage(usages) {
+  const costs = usages.map((usage2) => usage2.costUsd).filter((cost) => cost !== void 0);
+  const costComplete = !usages.some(
+    (usage2) => usage2.costUsd === void 0 && (usage2.totalTokens > 0 || usage2.toolCalls > 0)
+  );
   return {
     inputTokens: sum(usages, (usage2) => usage2.inputTokens),
     outputTokens: sum(usages, (usage2) => usage2.outputTokens),
     reasoningTokens: sum(usages, (usage2) => usage2.reasoningTokens),
     totalTokens: sum(usages, (usage2) => usage2.totalTokens),
+    ...costComplete && costs.length > 0 ? { costUsd: costs.reduce((total, cost) => total + cost, 0) } : {},
     toolCalls: sum(usages, (usage2) => usage2.toolCalls)
   };
 }