@vitest-evals/github-reporter 0.16.1 → 0.17.0

package/dist/index.js

@@ -128,7 +128,8 @@ function collectEvalReport(input, options = {}) {
   const cases = workspace.cases.map(collectEvalCase);
   const failures = cases.filter((testCase) => testCase.status === "failed");
   const evalScores = cases.map((testCase) => testCase.eval?.avgScore).filter((score) => isFiniteNumber(score));
-  const usage = sumUsage(cases);
+  const usage = sumAppUsage(cases);
+  const judgeUsage = sumJudgeUsage(cases);
   const durationMs = workspace.runs[0]?.durationMs;
   return {
     status: input.success && failures.length === 0 ? "passed" : "failed",
@@ -148,6 +149,7 @@ function collectEvalReport(input, options = {}) {
       minimum: Math.min(...evalScores)
     } : void 0,
     usage,
+    judgeUsage,
     cases,
     failures
   };
@@ -272,24 +274,67 @@ function stringifyReason(value) {
   }
   return typeof value === "string" ? value : stringifyValue(value, 4e3);
 }
-function sumUsage(cases) {
-  const usage = {
+function emptyUsage() {
+  return {
     inputTokens: 0,
     outputTokens: 0,
     reasoningTokens: 0,
     totalTokens: 0,
     toolCalls: 0
   };
+}
+function addRunUsage(total, usage) {
+  total.inputTokens += usage?.inputTokens ?? 0;
+  total.outputTokens += usage?.outputTokens ?? 0;
+  total.reasoningTokens += usage?.reasoningTokens ?? 0;
+  total.totalTokens += usage?.totalTokens ?? (usage?.inputTokens ?? 0) + (usage?.outputTokens ?? 0) + (usage?.reasoningTokens ?? 0);
+  if (usage?.costUsd !== void 0) {
+    total.costUsd = (total.costUsd ?? 0) + usage.costUsd;
+  }
+  total.toolCalls += usage?.toolCalls ?? 0;
+}
+function sumAppUsage(cases) {
+  const usage = emptyUsage();
+  const runUsages = cases.map(appUsageForCase).filter((item) => item !== void 0);
+  for (const runUsage of runUsages) {
+    addRunUsage(usage, runUsage);
+  }
+  omitPartialCost(usage, runUsages);
+  return usage;
+}
+function appUsageForCase(testCase) {
+  const usage = testCase.harness?.usage;
+  const effectiveToolCalls = toolCallCount(testCase);
+  if (!usage && effectiveToolCalls === 0) {
+    return void 0;
+  }
+  return {
+    ...usage,
+    ...effectiveToolCalls > 0 ? { toolCalls: effectiveToolCalls } : {}
+  };
+}
+function sumJudgeUsage(cases) {
+  const usage = emptyUsage();
+  const runUsages = [];
   for (const testCase of cases) {
-    const caseUsage = testCase.harness?.usage;
-    usage.inputTokens += caseUsage?.inputTokens ?? 0;
-    usage.outputTokens += caseUsage?.outputTokens ?? 0;
-    usage.reasoningTokens += caseUsage?.reasoningTokens ?? 0;
-    usage.totalTokens += caseUsage?.totalTokens ?? (caseUsage?.inputTokens ?? 0) + (caseUsage?.outputTokens ?? 0) + (caseUsage?.reasoningTokens ?? 0);
-    usage.toolCalls += toolCallCount(testCase);
+    for (const score of testCase.eval?.scores ?? []) {
+      for (const run of score.judgeRuns ?? []) {
+        runUsages.push(run.usage);
+        addRunUsage(usage, run.usage);
+      }
+    }
   }
+  omitPartialCost(usage, runUsages);
   return usage;
 }
+function omitPartialCost(total, usages) {
+  if (usages.some(hasUsageWithoutCost)) {
+    total.costUsd = void 0;
+  }
+}
+function hasUsageWithoutCost(usage) {
+  return usage?.costUsd === void 0 && ((usage?.totalTokens ?? (usage?.inputTokens ?? 0) + (usage?.outputTokens ?? 0) + (usage?.reasoningTokens ?? 0)) > 0 || (usage?.toolCalls ?? 0) > 0);
+}
 function toolCallCount(testCase) {
   const usageToolCalls = testCase.harness?.usage?.toolCalls;
   if (usageToolCalls !== void 0) {
@@ -433,8 +478,8 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate,
-      title: "Eval report hard failure",
-      message: `${formatNumber(nonEvalFailures)} non-eval test failure${nonEvalFailures === 1 ? "" : "s"}; ${counts}`
+      title: "Eval run failed",
+      message: `${formatNumber(nonEvalFailures)} test failure${nonEvalFailures === 1 ? "" : "s"} outside eval cases; ${counts}`
     };
   }
   if (report.totals.evalTotal === 0) {
@@ -443,7 +488,7 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate: null,
-      title: "Eval report hard failure",
+      title: "Eval run failed",
       message: "no eval cases were reported"
     };
   }
@@ -453,8 +498,8 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate,
-      title: "Eval report hard failure",
-      message: `vitest run failed without counted test failures; ${counts}`
+      title: "Eval run failed",
+      message: `Vitest failed without reporting a failed test; ${counts}`
     };
   }
   if (minPassRate !== void 0 && (passRate === null || passRate + Number.EPSILON < minPassRate)) {
@@ -464,7 +509,7 @@ function evaluateEvalGate(report, policy = {}) {
       enforced: true,
       passRate,
       title: `Eval pass rate ${formatPercent(passRate)} \u2014 required ${formatPercent(minPassRate)}`,
-      message: `eval pass rate below floor: ${counts}; required >= ${formatPercent(minPassRate)}`
+      message: `pass rate is below the minimum: ${counts}; minimum ${formatPercent(minPassRate)}`
     };
   }
   if (minScoreAverage !== void 0) {
@@ -475,8 +520,8 @@ function evaluateEvalGate(report, policy = {}) {
         status: "failed",
         enforced: true,
         passRate,
-        title: "Eval score gate failed",
-        message: `no score average available; required avg score >= ${formatScore(minScoreAverage)}`
+        title: "Average score unavailable",
+        message: `no average score was reported; minimum ${formatScore(minScoreAverage)}`
       };
     }
     if (average + Number.EPSILON < minScoreAverage) {
@@ -485,8 +530,8 @@ function evaluateEvalGate(report, policy = {}) {
         status: "failed",
         enforced: true,
         passRate,
-        title: `Avg score ${formatScore(average)} \u2014 required ${formatScore(minScoreAverage)}`,
-        message: `avg score below floor: ${counts}; required avg score >= ${formatScore(minScoreAverage)}`
+        title: `Average score ${formatScore(average)} \u2014 required ${formatScore(minScoreAverage)}`,
+        message: `average score is below the minimum: ${counts}; minimum ${formatScore(minScoreAverage)}`
       };
     }
   }
@@ -496,7 +541,7 @@ function evaluateEvalGate(report, policy = {}) {
     enforced: true,
     passRate,
     title: enforcedPassTitle(report, passRate, minPassRate, minScoreAverage),
-    message: `eval gate passed: ${counts}${formatFloorSuffix(minPassRate, minScoreAverage)}`
+    message: `requirements met: ${counts}${formatMinimumSuffix(minPassRate, minScoreAverage)}`
   };
 }
 function computePassRate(report) {
@@ -529,7 +574,7 @@ function resolveMinPassRate(policy) {
 }
 function defaultCheckTitle(report) {
   if (report.failures.length === 0 && report.status === "passed") {
-    return "No eval failures";
+    return "No eval cases failed";
   }
   if (report.failures.length === 0) {
     return "Vitest run failed";
@@ -538,10 +583,10 @@ function defaultCheckTitle(report) {
 }
 function enforcedPassTitle(report, passRate, minPassRate, minScoreAverage) {
   if (minPassRate !== void 0) {
-    return `Eval pass rate ${formatPercent(passRate)} \u2014 floor ${formatPercent(minPassRate)}`;
+    return `Eval pass rate ${formatPercent(passRate)} \u2014 minimum ${formatPercent(minPassRate)}`;
   }
   if (minScoreAverage !== void 0) {
-    return `Avg score ${formatScore(report.score?.average)} \u2014 floor ${formatScore(minScoreAverage)}`;
+    return `Average score ${formatScore(report.score?.average)} \u2014 minimum ${formatScore(minScoreAverage)}`;
   }
   return defaultCheckTitle(report);
 }
@@ -550,15 +595,15 @@ function formatEvalCounts(report, passRate) {
   const passRateText = passRate === null ? "n/a" : formatPercent(passRate);
   return `${formatNumber(report.totals.evalPassed)}/${formatNumber(
     report.totals.evalTotal
-  )} passed (${passRateText}), avg score ${scoreText}`;
+  )} passed (${passRateText}), average score ${scoreText}`;
 }
-function formatFloorSuffix(minPassRate, minScoreAverage) {
+function formatMinimumSuffix(minPassRate, minScoreAverage) {
   const parts = [];
   if (minPassRate !== void 0) {
-    parts.push(`pass rate floor ${formatPercent(minPassRate)}`);
+    parts.push(`minimum pass rate ${formatPercent(minPassRate)}`);
   }
   if (minScoreAverage !== void 0) {
-    parts.push(`avg score floor ${formatScore(minScoreAverage)}`);
+    parts.push(`minimum average score ${formatScore(minScoreAverage)}`);
   }
   return parts.length === 0 ? "" : `; ${parts.join(", ")}`;
 }
@@ -593,23 +638,23 @@ function renderJobSummary(report, options = {}) {
     ""
   ];
   if (report.failures.length > 0) {
-    const failureHeading = options.gate?.ok === true ? "### Quality Misses" : "### Failures";
+    const failureHeading = options.gate?.ok === true ? "### Cases Below Target" : "### Failures";
     lines.push(failureHeading, "");
     failures.forEach((testCase, index) => {
       lines.push(...renderFailureDetails(testCase, index + 1, options), "");
     });
     if (report.failures.length > failures.length) {
-      const omittedLabel = options.gate?.ok === true ? "quality misses" : "failures";
+      const omittedLabel = options.gate?.ok === true ? "cases below target" : "failures";
       lines.push(
         `${report.failures.length - failures.length} more ${omittedLabel} omitted from this summary.`,
         ""
       );
     }
   } else if (report.totals.evalTotal > 0) {
-    lines.push("### Failures", "", "No eval failures.", "");
+    lines.push("### Failures", "", "No eval cases failed.", "");
   }
   if (report.totals.evalTotal === 0) {
-    lines.push("No eval metadata was found in the Vitest JSON report.", "");
+    lines.push("No eval results were found in the Vitest JSON report.", "");
   }
   return `${lines.join("\n")}
 `;
@@ -619,9 +664,9 @@ function formatCountLine(passed, failed, total) {
 }
 function renderSummaryTable(report, nonEvalFailures, gate) {
   const rows = [
-    ["Status", gate?.status ?? report.status],
+    ["Status", capitalize(gate?.status ?? report.status)],
     [
-      "Evals",
+      "Eval cases",
       formatCountLine(
         report.totals.evalPassed,
         report.totals.evalFailed,
@@ -630,23 +675,31 @@ function renderSummaryTable(report, nonEvalFailures, gate) {
     ]
   ];
   if (gate?.passRate !== void 0 && gate.passRate !== null) {
-    rows.push(["Pass Rate", formatPercent(gate.passRate)]);
+    rows.push(["Pass rate", formatPercent(gate.passRate)]);
   } else if (report.totals.evalTotal > 0) {
     rows.push([
-      "Pass Rate",
+      "Pass rate",
       formatPercent(report.totals.evalPassed / report.totals.evalTotal)
     ]);
   }
   if (report.score) {
     rows.push(["Score", formatScoreSummary(report.score)]);
   }
+  if (hasUsage(report.usage) || hasUsage(report.judgeUsage)) {
+    rows.push(["App usage", formatUsage(report.usage)]);
+    rows.push(["Judge usage", formatUsage(report.judgeUsage)]);
+    rows.push([
+      "Total usage",
+      formatUsage(sumUsage(report.usage, report.judgeUsage))
+    ]);
+  }
   if (gate?.enforced) {
-    rows.push(["Gate", gate.message]);
+    rows.push(["Requirements", gate.message]);
   }
   if (nonEvalFailures > 0) {
     rows.push([
-      "Other Failures",
-      `${formatNumber(nonEvalFailures)} non-eval test failure${nonEvalFailures === 1 ? "" : "s"}`
+      "Other test failures",
+      `${formatNumber(nonEvalFailures)} test failure${nonEvalFailures === 1 ? "" : "s"} outside eval cases`
     ]);
   }
   rows.push(["Duration", formatDuration(report.durationMs)]);
@@ -659,7 +712,53 @@ function renderSummaryTable(report, nonEvalFailures, gate) {
   ];
 }
 function formatScoreSummary(score) {
-  return `avg ${formatScore(score.average)}${score.minimum === void 0 ? "" : `, min ${formatScore(score.minimum)}`}`;
+  return `average ${formatScore(score.average)}${score.minimum === void 0 ? "" : `, lowest ${formatScore(score.minimum)}`}`;
+}
+function capitalize(value) {
+  return value.charAt(0).toUpperCase() + value.slice(1);
+}
+function hasUsage(usage) {
+  return usage.totalTokens > 0 || usage.costUsd !== void 0 || usage.toolCalls > 0;
+}
+function sumUsage(app, judge) {
+  const appCostKnown = !hasUsageWithoutCost2(app);
+  const judgeCostKnown = !hasUsageWithoutCost2(judge);
+  return {
+    inputTokens: app.inputTokens + judge.inputTokens,
+    outputTokens: app.outputTokens + judge.outputTokens,
+    reasoningTokens: app.reasoningTokens + judge.reasoningTokens,
+    totalTokens: app.totalTokens + judge.totalTokens,
+    ...appCostKnown && judgeCostKnown ? { costUsd: (app.costUsd ?? 0) + (judge.costUsd ?? 0) } : {},
+    toolCalls: app.toolCalls + judge.toolCalls
+  };
+}
+function hasUsageWithoutCost2(usage) {
+  return usage.costUsd === void 0 && (usage.totalTokens > 0 || usage.toolCalls > 0);
+}
+function formatUsage(usage) {
+  const parts = [];
+  if (usage.totalTokens > 0) {
+    parts.push(`${formatNumber(usage.totalTokens)} tokens`);
+  }
+  if (usage.costUsd !== void 0) {
+    parts.push(
+      usage.costUsd.toLocaleString("en-US", {
+        style: "currency",
+        currency: "USD",
+        minimumFractionDigits: 2,
+        maximumFractionDigits: 6
+      })
+    );
+  }
+  if (hasUsageWithoutCost2(usage)) {
+    parts.push("cost unavailable");
+  }
+  if (usage.toolCalls > 0) {
+    parts.push(
+      `${formatNumber(usage.toolCalls)} tool call${usage.toolCalls === 1 ? "" : "s"}`
+    );
+  }
+  return parts.join(", ") || "none";
 }
 function escapeTableCell(value) {
   return value.replace(/\r?\n/g, " ").replace(/\\/g, "\\\\").replace(/\|/g, "\\|");
@@ -739,7 +838,7 @@ function renderFailureBlock(testCase, {
     ["Case", `${number}. ${testCase.displayName}`],
     ["Status", testCase.status],
     ["Location", formatLocation(testCase.displayFile, testCase.location)],
-    ["Harness", testCase.harness?.name ?? "n/a"],
+    ["App runner", testCase.harness?.name ?? "n/a"],
     ["Score", formatScore(failure?.score ?? testCase.eval?.avgScore)],
     ["Judge", failure?.judgeName ?? "n/a"]
   ];
@@ -777,7 +876,7 @@ function renderFailureBlock(testCase, {
   if (finalOutput !== void 0) {
     lines.push(
       ...renderAsciiSection(
-        "Final Output",
+        "Output",
         stringifyValue(finalOutput, maxOutputChars).split(/\r?\n/)
       ),
       ""
@@ -805,7 +904,7 @@ function renderFailureBlock(testCase, {
   if (testCase.harness?.errors.length) {
     lines.push(
       ...renderAsciiSection(
-        "Harness Errors",
+        "App errors",
         stringifyValue(testCase.harness.errors, maxReasonChars).split(/\r?\n/)
       ),
       ""
@@ -854,7 +953,9 @@ function formatCaseUsage(testCase) {
     parts.push(`${formatNumber(totalTokens)} tokens`);
   }
   if (toolCalls > 0) {
-    parts.push(`${formatNumber(toolCalls)} tool${toolCalls === 1 ? "" : "s"}`);
+    parts.push(
+      `${formatNumber(toolCalls)} tool call${toolCalls === 1 ? "" : "s"}`
+    );
   }
   if (testCase.harness?.timingMs !== void 0) {
     parts.push(formatDuration(testCase.harness.timingMs));
@@ -1024,16 +1125,22 @@ function mergeEvalReports(reports) {
       minimum: Math.min(...scoredCases)
     } : void 0,
     usage: mergeUsage(reports.map((report) => report.usage)),
+    judgeUsage: mergeUsage(reports.map((report) => report.judgeUsage)),
     cases,
     failures
   };
 }
 function mergeUsage(usages) {
+  const costs = usages.map((usage) => usage.costUsd).filter((cost) => cost !== void 0);
+  const costComplete = !usages.some(
+    (usage) => usage.costUsd === void 0 && (usage.totalTokens > 0 || usage.toolCalls > 0)
+  );
   return {
     inputTokens: sum(usages, (usage) => usage.inputTokens),
     outputTokens: sum(usages, (usage) => usage.outputTokens),
     reasoningTokens: sum(usages, (usage) => usage.reasoningTokens),
     totalTokens: sum(usages, (usage) => usage.totalTokens),
+    ...costComplete && costs.length > 0 ? { costUsd: costs.reduce((total, cost) => total + cost, 0) } : {},
     toolCalls: sum(usages, (usage) => usage.toolCalls)
   };
 }