@vitest-evals/github-reporter 0.16.0 → 0.17.0

package/dist/index.mjs

@@ -93,7 +93,8 @@ function collectEvalReport(input, options = {}) {
   const cases = workspace.cases.map(collectEvalCase);
   const failures = cases.filter((testCase) => testCase.status === "failed");
   const evalScores = cases.map((testCase) => testCase.eval?.avgScore).filter((score) => isFiniteNumber(score));
-  const usage = sumUsage(cases);
+  const usage = sumAppUsage(cases);
+  const judgeUsage = sumJudgeUsage(cases);
   const durationMs = workspace.runs[0]?.durationMs;
   return {
     status: input.success && failures.length === 0 ? "passed" : "failed",
@@ -113,6 +114,7 @@ function collectEvalReport(input, options = {}) {
       minimum: Math.min(...evalScores)
     } : void 0,
     usage,
+    judgeUsage,
     cases,
     failures
   };
@@ -237,24 +239,67 @@ function stringifyReason(value) {
   }
   return typeof value === "string" ? value : stringifyValue(value, 4e3);
 }
-function sumUsage(cases) {
-  const usage = {
+function emptyUsage() {
+  return {
     inputTokens: 0,
     outputTokens: 0,
     reasoningTokens: 0,
     totalTokens: 0,
     toolCalls: 0
   };
+}
+function addRunUsage(total, usage) {
+  total.inputTokens += usage?.inputTokens ?? 0;
+  total.outputTokens += usage?.outputTokens ?? 0;
+  total.reasoningTokens += usage?.reasoningTokens ?? 0;
+  total.totalTokens += usage?.totalTokens ?? (usage?.inputTokens ?? 0) + (usage?.outputTokens ?? 0) + (usage?.reasoningTokens ?? 0);
+  if (usage?.costUsd !== void 0) {
+    total.costUsd = (total.costUsd ?? 0) + usage.costUsd;
+  }
+  total.toolCalls += usage?.toolCalls ?? 0;
+}
+function sumAppUsage(cases) {
+  const usage = emptyUsage();
+  const runUsages = cases.map(appUsageForCase).filter((item) => item !== void 0);
+  for (const runUsage of runUsages) {
+    addRunUsage(usage, runUsage);
+  }
+  omitPartialCost(usage, runUsages);
+  return usage;
+}
+function appUsageForCase(testCase) {
+  const usage = testCase.harness?.usage;
+  const effectiveToolCalls = toolCallCount(testCase);
+  if (!usage && effectiveToolCalls === 0) {
+    return void 0;
+  }
+  return {
+    ...usage,
+    ...effectiveToolCalls > 0 ? { toolCalls: effectiveToolCalls } : {}
+  };
+}
+function sumJudgeUsage(cases) {
+  const usage = emptyUsage();
+  const runUsages = [];
   for (const testCase of cases) {
-    const caseUsage = testCase.harness?.usage;
-    usage.inputTokens += caseUsage?.inputTokens ?? 0;
-    usage.outputTokens += caseUsage?.outputTokens ?? 0;
-    usage.reasoningTokens += caseUsage?.reasoningTokens ?? 0;
-    usage.totalTokens += caseUsage?.totalTokens ?? (caseUsage?.inputTokens ?? 0) + (caseUsage?.outputTokens ?? 0) + (caseUsage?.reasoningTokens ?? 0);
-    usage.toolCalls += toolCallCount(testCase);
+    for (const score of testCase.eval?.scores ?? []) {
+      for (const run of score.judgeRuns ?? []) {
+        runUsages.push(run.usage);
+        addRunUsage(usage, run.usage);
+      }
+    }
   }
+  omitPartialCost(usage, runUsages);
   return usage;
 }
+function omitPartialCost(total, usages) {
+  if (usages.some(hasUsageWithoutCost)) {
+    total.costUsd = void 0;
+  }
+}
+function hasUsageWithoutCost(usage) {
+  return usage?.costUsd === void 0 && ((usage?.totalTokens ?? (usage?.inputTokens ?? 0) + (usage?.outputTokens ?? 0) + (usage?.reasoningTokens ?? 0)) > 0 || (usage?.toolCalls ?? 0) > 0);
+}
 function toolCallCount(testCase) {
   const usageToolCalls = testCase.harness?.usage?.toolCalls;
   if (usageToolCalls !== void 0) {
@@ -398,8 +443,8 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate,
-      title: "Eval report hard failure",
-      message: `${formatNumber(nonEvalFailures)} non-eval test failure${nonEvalFailures === 1 ? "" : "s"}; ${counts}`
+      title: "Eval run failed",
+      message: `${formatNumber(nonEvalFailures)} test failure${nonEvalFailures === 1 ? "" : "s"} outside eval cases; ${counts}`
     };
   }
   if (report.totals.evalTotal === 0) {
@@ -408,7 +453,7 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate: null,
-      title: "Eval report hard failure",
+      title: "Eval run failed",
       message: "no eval cases were reported"
     };
   }
@@ -418,8 +463,8 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate,
-      title: "Eval report hard failure",
-      message: `vitest run failed without counted test failures; ${counts}`
+      title: "Eval run failed",
+      message: `Vitest failed without reporting a failed test; ${counts}`
     };
   }
   if (minPassRate !== void 0 && (passRate === null || passRate + Number.EPSILON < minPassRate)) {
@@ -429,7 +474,7 @@ function evaluateEvalGate(report, policy = {}) {
       enforced: true,
       passRate,
       title: `Eval pass rate ${formatPercent(passRate)} \u2014 required ${formatPercent(minPassRate)}`,
-      message: `eval pass rate below floor: ${counts}; required >= ${formatPercent(minPassRate)}`
+      message: `pass rate is below the minimum: ${counts}; minimum ${formatPercent(minPassRate)}`
     };
   }
   if (minScoreAverage !== void 0) {
@@ -440,8 +485,8 @@ function evaluateEvalGate(report, policy = {}) {
         status: "failed",
         enforced: true,
         passRate,
-        title: "Eval score gate failed",
-        message: `no score average available; required avg score >= ${formatScore(minScoreAverage)}`
+        title: "Average score unavailable",
+        message: `no average score was reported; minimum ${formatScore(minScoreAverage)}`
       };
     }
     if (average + Number.EPSILON < minScoreAverage) {
@@ -450,8 +495,8 @@ function evaluateEvalGate(report, policy = {}) {
         status: "failed",
         enforced: true,
         passRate,
-        title: `Avg score ${formatScore(average)} \u2014 required ${formatScore(minScoreAverage)}`,
-        message: `avg score below floor: ${counts}; required avg score >= ${formatScore(minScoreAverage)}`
+        title: `Average score ${formatScore(average)} \u2014 required ${formatScore(minScoreAverage)}`,
+        message: `average score is below the minimum: ${counts}; minimum ${formatScore(minScoreAverage)}`
       };
     }
   }
@@ -461,7 +506,7 @@ function evaluateEvalGate(report, policy = {}) {
     enforced: true,
     passRate,
     title: enforcedPassTitle(report, passRate, minPassRate, minScoreAverage),
-    message: `eval gate passed: ${counts}${formatFloorSuffix(minPassRate, minScoreAverage)}`
+    message: `requirements met: ${counts}${formatMinimumSuffix(minPassRate, minScoreAverage)}`
   };
 }
 function computePassRate(report) {
@@ -494,7 +539,7 @@ function resolveMinPassRate(policy) {
 }
 function defaultCheckTitle(report) {
   if (report.failures.length === 0 && report.status === "passed") {
-    return "No eval failures";
+    return "No eval cases failed";
   }
   if (report.failures.length === 0) {
     return "Vitest run failed";
@@ -503,10 +548,10 @@ function defaultCheckTitle(report) {
 }
 function enforcedPassTitle(report, passRate, minPassRate, minScoreAverage) {
   if (minPassRate !== void 0) {
-    return `Eval pass rate ${formatPercent(passRate)} \u2014 floor ${formatPercent(minPassRate)}`;
+    return `Eval pass rate ${formatPercent(passRate)} \u2014 minimum ${formatPercent(minPassRate)}`;
   }
   if (minScoreAverage !== void 0) {
-    return `Avg score ${formatScore(report.score?.average)} \u2014 floor ${formatScore(minScoreAverage)}`;
+    return `Average score ${formatScore(report.score?.average)} \u2014 minimum ${formatScore(minScoreAverage)}`;
   }
   return defaultCheckTitle(report);
 }
@@ -515,19 +560,22 @@ function formatEvalCounts(report, passRate) {
   const passRateText = passRate === null ? "n/a" : formatPercent(passRate);
   return `${formatNumber(report.totals.evalPassed)}/${formatNumber(
     report.totals.evalTotal
-  )} passed (${passRateText}), avg score ${scoreText}`;
+  )} passed (${passRateText}), average score ${scoreText}`;
 }
-function formatFloorSuffix(minPassRate, minScoreAverage) {
+function formatMinimumSuffix(minPassRate, minScoreAverage) {
   const parts = [];
   if (minPassRate !== void 0) {
-    parts.push(`pass rate floor ${formatPercent(minPassRate)}`);
+    parts.push(`minimum pass rate ${formatPercent(minPassRate)}`);
   }
   if (minScoreAverage !== void 0) {
-    parts.push(`avg score floor ${formatScore(minScoreAverage)}`);
+    parts.push(`minimum average score ${formatScore(minScoreAverage)}`);
   }
   return parts.length === 0 ? "" : `; ${parts.join(", ")}`;
 }
 
+// src/github.ts
+import { readFileSync } from "fs";
+
 // src/summary.ts
 var DEFAULT_MAX_FAILURES = 20;
 var DEFAULT_MAX_REASON_CHARS = 8e3;
@@ -555,23 +603,23 @@ function renderJobSummary(report, options = {}) {
     ""
   ];
   if (report.failures.length > 0) {
-    const failureHeading = options.gate?.ok === true ? "### Quality Misses" : "### Failures";
+    const failureHeading = options.gate?.ok === true ? "### Cases Below Target" : "### Failures";
     lines.push(failureHeading, "");
     failures.forEach((testCase, index) => {
       lines.push(...renderFailureDetails(testCase, index + 1, options), "");
     });
     if (report.failures.length > failures.length) {
-      const omittedLabel = options.gate?.ok === true ? "quality misses" : "failures";
+      const omittedLabel = options.gate?.ok === true ? "cases below target" : "failures";
       lines.push(
         `${report.failures.length - failures.length} more ${omittedLabel} omitted from this summary.`,
         ""
       );
     }
   } else if (report.totals.evalTotal > 0) {
-    lines.push("### Failures", "", "No eval failures.", "");
+    lines.push("### Failures", "", "No eval cases failed.", "");
   }
   if (report.totals.evalTotal === 0) {
-    lines.push("No eval metadata was found in the Vitest JSON report.", "");
+    lines.push("No eval results were found in the Vitest JSON report.", "");
   }
   return `${lines.join("\n")}
 `;
@@ -581,9 +629,9 @@ function formatCountLine(passed, failed, total) {
 }
 function renderSummaryTable(report, nonEvalFailures, gate) {
   const rows = [
-    ["Status", gate?.status ?? report.status],
+    ["Status", capitalize(gate?.status ?? report.status)],
     [
-      "Evals",
+      "Eval cases",
       formatCountLine(
         report.totals.evalPassed,
         report.totals.evalFailed,
@@ -592,23 +640,31 @@ function renderSummaryTable(report, nonEvalFailures, gate) {
     ]
   ];
   if (gate?.passRate !== void 0 && gate.passRate !== null) {
-    rows.push(["Pass Rate", formatPercent(gate.passRate)]);
+    rows.push(["Pass rate", formatPercent(gate.passRate)]);
   } else if (report.totals.evalTotal > 0) {
     rows.push([
-      "Pass Rate",
+      "Pass rate",
       formatPercent(report.totals.evalPassed / report.totals.evalTotal)
     ]);
   }
   if (report.score) {
     rows.push(["Score", formatScoreSummary(report.score)]);
   }
+  if (hasUsage(report.usage) || hasUsage(report.judgeUsage)) {
+    rows.push(["App usage", formatUsage(report.usage)]);
+    rows.push(["Judge usage", formatUsage(report.judgeUsage)]);
+    rows.push([
+      "Total usage",
+      formatUsage(sumUsage(report.usage, report.judgeUsage))
+    ]);
+  }
   if (gate?.enforced) {
-    rows.push(["Gate", gate.message]);
+    rows.push(["Requirements", gate.message]);
   }
   if (nonEvalFailures > 0) {
     rows.push([
-      "Other Failures",
-      `${formatNumber(nonEvalFailures)} non-eval test failure${nonEvalFailures === 1 ? "" : "s"}`
+      "Other test failures",
+      `${formatNumber(nonEvalFailures)} test failure${nonEvalFailures === 1 ? "" : "s"} outside eval cases`
     ]);
   }
   rows.push(["Duration", formatDuration(report.durationMs)]);
@@ -621,7 +677,53 @@ function renderSummaryTable(report, nonEvalFailures, gate) {
   ];
 }
 function formatScoreSummary(score) {
-  return `avg ${formatScore(score.average)}${score.minimum === void 0 ? "" : `, min ${formatScore(score.minimum)}`}`;
+  return `average ${formatScore(score.average)}${score.minimum === void 0 ? "" : `, lowest ${formatScore(score.minimum)}`}`;
+}
+function capitalize(value) {
+  return value.charAt(0).toUpperCase() + value.slice(1);
+}
+function hasUsage(usage) {
+  return usage.totalTokens > 0 || usage.costUsd !== void 0 || usage.toolCalls > 0;
+}
+function sumUsage(app, judge) {
+  const appCostKnown = !hasUsageWithoutCost2(app);
+  const judgeCostKnown = !hasUsageWithoutCost2(judge);
+  return {
+    inputTokens: app.inputTokens + judge.inputTokens,
+    outputTokens: app.outputTokens + judge.outputTokens,
+    reasoningTokens: app.reasoningTokens + judge.reasoningTokens,
+    totalTokens: app.totalTokens + judge.totalTokens,
+    ...appCostKnown && judgeCostKnown ? { costUsd: (app.costUsd ?? 0) + (judge.costUsd ?? 0) } : {},
+    toolCalls: app.toolCalls + judge.toolCalls
+  };
+}
+function hasUsageWithoutCost2(usage) {
+  return usage.costUsd === void 0 && (usage.totalTokens > 0 || usage.toolCalls > 0);
+}
+function formatUsage(usage) {
+  const parts = [];
+  if (usage.totalTokens > 0) {
+    parts.push(`${formatNumber(usage.totalTokens)} tokens`);
+  }
+  if (usage.costUsd !== void 0) {
+    parts.push(
+      usage.costUsd.toLocaleString("en-US", {
+        style: "currency",
+        currency: "USD",
+        minimumFractionDigits: 2,
+        maximumFractionDigits: 6
+      })
+    );
+  }
+  if (hasUsageWithoutCost2(usage)) {
+    parts.push("cost unavailable");
+  }
+  if (usage.toolCalls > 0) {
+    parts.push(
+      `${formatNumber(usage.toolCalls)} tool call${usage.toolCalls === 1 ? "" : "s"}`
+    );
+  }
+  return parts.join(", ") || "none";
 }
 function escapeTableCell(value) {
   return value.replace(/\r?\n/g, " ").replace(/\\/g, "\\\\").replace(/\|/g, "\\|");
@@ -701,7 +803,7 @@ function renderFailureBlock(testCase, {
     ["Case", `${number}. ${testCase.displayName}`],
     ["Status", testCase.status],
     ["Location", formatLocation(testCase.displayFile, testCase.location)],
-    ["Harness", testCase.harness?.name ?? "n/a"],
+    ["App runner", testCase.harness?.name ?? "n/a"],
     ["Score", formatScore(failure?.score ?? testCase.eval?.avgScore)],
     ["Judge", failure?.judgeName ?? "n/a"]
   ];
@@ -739,7 +841,7 @@ function renderFailureBlock(testCase, {
   if (finalOutput !== void 0) {
     lines.push(
       ...renderAsciiSection(
-        "Final Output",
+        "Output",
         stringifyValue(finalOutput, maxOutputChars).split(/\r?\n/)
       ),
       ""
@@ -767,7 +869,7 @@ function renderFailureBlock(testCase, {
   if (testCase.harness?.errors.length) {
     lines.push(
       ...renderAsciiSection(
-        "Harness Errors",
+        "App errors",
         stringifyValue(testCase.harness.errors, maxReasonChars).split(/\r?\n/)
       ),
       ""
@@ -816,7 +918,9 @@ function formatCaseUsage(testCase) {
     parts.push(`${formatNumber(totalTokens)} tokens`);
   }
   if (toolCalls > 0) {
-    parts.push(`${formatNumber(toolCalls)} tool${toolCalls === 1 ? "" : "s"}`);
+    parts.push(
+      `${formatNumber(toolCalls)} tool call${toolCalls === 1 ? "" : "s"}`
+    );
   }
   if (testCase.harness?.timingMs !== void 0) {
     parts.push(formatDuration(testCase.harness.timingMs));
@@ -828,10 +932,44 @@ function formatCaseUsage(testCase) {
 var DEFAULT_CHECK_NAME = "vitest-evals";
 var MAX_CHECK_SUMMARY_LENGTH = 64e3;
 var CHECK_SUMMARY_TRUNCATION_SUFFIX = "\n\n[truncated for GitHub Check Run]\n";
+function resolveCheckSha(env = process.env, options = {}) {
+  const explicit = options.sha?.trim() || env.GITHUB_PR_HEAD_SHA?.trim();
+  if (explicit) {
+    return explicit;
+  }
+  const eventPath = options.eventPath?.trim() || env.GITHUB_EVENT_PATH?.trim();
+  if (eventPath) {
+    try {
+      const event = JSON.parse(readFileSync(eventPath, "utf8"));
+      const headSha = event.pull_request?.head?.sha;
+      if (typeof headSha === "string" && headSha.trim()) {
+        return headSha.trim();
+      }
+    } catch {
+    }
+  }
+  return env.GITHUB_SHA?.trim() || void 0;
+}
+function resolveCheckDetailsUrl(env = process.env, options = {}) {
+  const explicit = options.detailsUrl?.trim();
+  if (explicit) {
+    return explicit;
+  }
+  const server = env.GITHUB_SERVER_URL?.replace(/\/$/, "");
+  const repository = env.GITHUB_REPOSITORY?.trim();
+  const runId = env.GITHUB_RUN_ID?.trim();
+  if (!server || !repository || !runId) {
+    return void 0;
+  }
+  return `${server}/${repository}/actions/runs/${runId}`;
+}
 async function publishCheckRun(report, options = {}) {
   const token = options.token ?? process.env.GITHUB_TOKEN;
   const repository = options.repository ?? process.env.GITHUB_REPOSITORY;
-  const sha = options.sha ?? process.env.GITHUB_SHA;
+  const sha = resolveCheckSha(process.env, { sha: options.sha });
+  const detailsUrl = resolveCheckDetailsUrl(process.env, {
+    detailsUrl: options.detailsUrl
+  });
   if (!token) {
     return { status: "skipped", reason: "missing GITHUB_TOKEN" };
   }
@@ -839,7 +977,10 @@ async function publishCheckRun(report, options = {}) {
     return { status: "skipped", reason: "missing GITHUB_REPOSITORY" };
   }
   if (!sha && options.checkRunId === void 0) {
-    return { status: "skipped", reason: "missing GITHUB_SHA" };
+    return {
+      status: "skipped",
+      reason: "missing commit SHA (set --sha / options.sha, GITHUB_PR_HEAD_SHA, pull_request.head.sha, or GITHUB_SHA)"
+    };
   }
   const [owner, repo] = repository.split("/");
   if (!owner || !repo) {
@@ -848,7 +989,7 @@ async function publishCheckRun(report, options = {}) {
       reason: `invalid GitHub repository: ${repository}`
     };
   }
-  const payload = buildCheckRunPayload(report, options);
+  const payload = buildCheckRunPayload(report, options, detailsUrl);
   const apiUrl = options.apiUrl ?? process.env.GITHUB_API_URL ?? "https://api.github.com";
   const requestUrl = options.checkRunId === void 0 ? `${apiUrl}/repos/${owner}/${repo}/check-runs` : `${apiUrl}/repos/${owner}/${repo}/check-runs/${options.checkRunId}`;
   const response = await fetch(requestUrl, {
@@ -863,6 +1004,7 @@ async function publishCheckRun(report, options = {}) {
       options.checkRunId === void 0 ? {
         name: options.name ?? DEFAULT_CHECK_NAME,
         head_sha: sha,
+        ...options.externalId ? { external_id: options.externalId } : {},
         ...payload
       } : payload
     )
@@ -877,10 +1019,11 @@ async function publishCheckRun(report, options = {}) {
   return {
     status: options.checkRunId === void 0 ? "created" : "updated",
     id: data.id,
-    htmlUrl: data.html_url
+    htmlUrl: data.html_url,
+    sha
   };
 }
-function buildCheckRunPayload(report, options) {
+function buildCheckRunPayload(report, options, detailsUrl) {
   const gate = options.gate ?? evaluateEvalGate(report);
   const annotations = buildCheckAnnotations(report, {
     maxAnnotations: options.maxAnnotations,
@@ -890,6 +1033,7 @@ function buildCheckRunPayload(report, options) {
     status: "completed",
     conclusion: gate.ok ? "success" : "failure",
     completed_at: (/* @__PURE__ */ new Date()).toISOString(),
+    ...detailsUrl ? { details_url: detailsUrl } : {},
     output: {
       title: gate.title,
       summary: truncateCheckSummary(
@@ -949,16 +1093,22 @@ function mergeEvalReports(reports) {
       minimum: Math.min(...scoredCases)
     } : void 0,
     usage: mergeUsage(reports.map((report) => report.usage)),
+    judgeUsage: mergeUsage(reports.map((report) => report.judgeUsage)),
     cases,
     failures
   };
 }
 function mergeUsage(usages) {
+  const costs = usages.map((usage) => usage.costUsd).filter((cost) => cost !== void 0);
+  const costComplete = !usages.some(
+    (usage) => usage.costUsd === void 0 && (usage.totalTokens > 0 || usage.toolCalls > 0)
+  );
   return {
     inputTokens: sum(usages, (usage) => usage.inputTokens),
     outputTokens: sum(usages, (usage) => usage.outputTokens),
     reasoningTokens: sum(usages, (usage) => usage.reasoningTokens),
     totalTokens: sum(usages, (usage) => usage.totalTokens),
+    ...costComplete && costs.length > 0 ? { costUsd: costs.reduce((total, cost) => total + cost, 0) } : {},
     toolCalls: sum(usages, (usage) => usage.toolCalls)
   };
 }
@@ -1050,11 +1200,17 @@ async function publishEvalReport(options) {
         name: options.checkName,
         repository: options.repository,
         sha: options.sha,
+        detailsUrl: options.detailsUrl,
+        externalId: options.externalId,
         token: options.token,
         gate
       });
       if (checkRun.status === "skipped") {
         options.warn?.(`GitHub Check Run skipped: ${checkRun.reason}`);
+      } else if (checkRun.htmlUrl) {
+        console.log(`published check run: ${checkRun.htmlUrl}`);
+      } else if (checkRun.id !== void 0) {
+        console.log(`published check run id: ${checkRun.id}`);
       }
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -1064,14 +1220,21 @@ async function publishEvalReport(options) {
       options.warn?.(message);
     }
   }
+  const gateFailed = gate.enforced && !gate.ok;
+  const wantsSoftFail = options.softFail ?? options.checkRun === true;
+  const softFail = wantsSoftFail && checkRunPublished(checkRun);
+  const shouldFail = gateFailed && !softFail;
   return {
     report,
     resultFiles,
     gate,
-    shouldFail: gate.enforced && !gate.ok,
+    shouldFail,
     checkRun
   };
 }
+function checkRunPublished(checkRun) {
+  return checkRun?.status === "created" || checkRun?.status === "updated";
+}
 export {
   buildCheckAnnotations,
   collectEvalReport,
@@ -1082,6 +1245,8 @@ export {
   publishEvalReport,
   renderGateWorkflowCommand,
   renderJobSummary,
-  renderWorkflowCommands
+  renderWorkflowCommands,
+  resolveCheckDetailsUrl,
+  resolveCheckSha
 };
 //# sourceMappingURL=index.mjs.map