@vitest-evals/github-reporter 0.16.0 → 0.17.0

package/README.md

@@ -31,10 +31,19 @@ steps:
     with:
       results: vitest-results.json
       publish-check: true
+      min-pass-rate: 0.8
 ```
 
+On `pull_request`, the Check Run attaches to the PR head SHA (not the temporary
+merge `GITHUB_SHA`) so it shows on the PR checks list. Override with the `sha`
+input when needed.
+
+When a Check Run publishes successfully with a gate, the action step soft-fails
+by default so the Check Run owns green/red. If publishing is skipped, the step
+still fails on a rejected gate. Set `soft-fail: false` to also fail the job.
+
 If configuration or permission is missing, the action keeps the job summary and
-workflow annotations and warns instead of failing.
+workflow annotations and warns instead of failing solely for the missing check.
 
 ## Score and Pass-Rate Gates
 
@@ -49,9 +58,11 @@ workflow annotations and warns instead of failing.
 
 - `fail-on-failures: true` requires every eval case to pass
 - `min-pass-rate` and `min-score-average` set aggregate floors in the `0`-`1` range
-- `status`, Check Run conclusion/title, and step exit follow the gate
+- `status` and Check Run conclusion/title follow the gate
 - quality misses become warnings when the gate still passes
 - non-eval / infrastructure failures still fail hard
+- published Check Runs soft-fail the step by default; set `soft-fail: false` to
+  also fail the workflow job
 - use `evals-failed` / `pass-rate` for raw tallies (`pass-rate` is a 0-1 ratio)
 
 ## Sharded Reports
@@ -80,10 +91,12 @@ final reducer job:
 | `results` | `vitest-results.json` | Vitest JSON result files. Supports paths, `*` and `**` globs, and newline-separated entries. |
 | `publish-summary` | `true` | Write a GitHub Actions job summary. |
 | `publish-annotations` | `true` | Emit GitHub workflow annotations for failed evals. |
-| `publish-check` | `false` | Publish one GitHub Check Run for the combined report. |
+| `publish-check` | `false` | Publish one GitHub Check Run for the combined report. Attaches to PR head on `pull_request`. |
 | `check-name` | `vitest-evals` | Name of the GitHub Check Run. |
 | `github-token` | `${{ github.token }}` | Token used for Check Run publishing. |
+| `sha` | PR head, else `GITHUB_SHA` | Commit SHA for the Check Run. |
 | `fail-on-failures` | `false` | Fail the action when any eval case failed. Equivalent to `min-pass-rate: 1`. |
+| `soft-fail` | auto | Keep the step green when a published Check Run owns a failed gate. |
 | `min-pass-rate` | unset | Minimum fraction of eval cases that must pass (`0`-`1`). |
 | `min-score-average` | unset | Minimum average eval score across scored cases (`0`-`1`). |
 | `max-annotations` | unset | Maximum number of failure annotations to publish. Check Run annotations are capped at 50 by GitHub. |

package/dist/cli.js

@@ -41,6 +41,12 @@ function parseCliArgs(args, env = process.env) {
       case "--fail-on-check-error":
         options.failOnCheckError = true;
         break;
+      case "--soft-fail":
+        options.softFail = true;
+        break;
+      case "--no-soft-fail":
+        options.softFail = false;
+        break;
       case "--min-pass-rate":
         options.minPassRate = readRatio(args, ++index, arg);
         break;
@@ -317,7 +323,8 @@ function collectEvalReport(input, options = {}) {
   const cases = workspace.cases.map(collectEvalCase);
   const failures = cases.filter((testCase) => testCase.status === "failed");
   const evalScores = cases.map((testCase) => testCase.eval?.avgScore).filter((score) => isFiniteNumber(score));
-  const usage2 = sumUsage(cases);
+  const usage2 = sumAppUsage(cases);
+  const judgeUsage = sumJudgeUsage(cases);
   const durationMs = workspace.runs[0]?.durationMs;
   return {
     status: input.success && failures.length === 0 ? "passed" : "failed",
@@ -337,6 +344,7 @@ function collectEvalReport(input, options = {}) {
       minimum: Math.min(...evalScores)
     } : void 0,
     usage: usage2,
+    judgeUsage,
     cases,
     failures
   };
@@ -461,24 +469,67 @@ function stringifyReason(value) {
   }
   return typeof value === "string" ? value : stringifyValue(value, 4e3);
 }
-function sumUsage(cases) {
-  const usage2 = {
+function emptyUsage() {
+  return {
     inputTokens: 0,
     outputTokens: 0,
     reasoningTokens: 0,
     totalTokens: 0,
     toolCalls: 0
   };
+}
+function addRunUsage(total, usage2) {
+  total.inputTokens += usage2?.inputTokens ?? 0;
+  total.outputTokens += usage2?.outputTokens ?? 0;
+  total.reasoningTokens += usage2?.reasoningTokens ?? 0;
+  total.totalTokens += usage2?.totalTokens ?? (usage2?.inputTokens ?? 0) + (usage2?.outputTokens ?? 0) + (usage2?.reasoningTokens ?? 0);
+  if (usage2?.costUsd !== void 0) {
+    total.costUsd = (total.costUsd ?? 0) + usage2.costUsd;
+  }
+  total.toolCalls += usage2?.toolCalls ?? 0;
+}
+function sumAppUsage(cases) {
+  const usage2 = emptyUsage();
+  const runUsages = cases.map(appUsageForCase).filter((item) => item !== void 0);
+  for (const runUsage of runUsages) {
+    addRunUsage(usage2, runUsage);
+  }
+  omitPartialCost(usage2, runUsages);
+  return usage2;
+}
+function appUsageForCase(testCase) {
+  const usage2 = testCase.harness?.usage;
+  const effectiveToolCalls = toolCallCount(testCase);
+  if (!usage2 && effectiveToolCalls === 0) {
+    return void 0;
+  }
+  return {
+    ...usage2,
+    ...effectiveToolCalls > 0 ? { toolCalls: effectiveToolCalls } : {}
+  };
+}
+function sumJudgeUsage(cases) {
+  const usage2 = emptyUsage();
+  const runUsages = [];
   for (const testCase of cases) {
-    const caseUsage = testCase.harness?.usage;
-    usage2.inputTokens += caseUsage?.inputTokens ?? 0;
-    usage2.outputTokens += caseUsage?.outputTokens ?? 0;
-    usage2.reasoningTokens += caseUsage?.reasoningTokens ?? 0;
-    usage2.totalTokens += caseUsage?.totalTokens ?? (caseUsage?.inputTokens ?? 0) + (caseUsage?.outputTokens ?? 0) + (caseUsage?.reasoningTokens ?? 0);
-    usage2.toolCalls += toolCallCount(testCase);
+    for (const score of testCase.eval?.scores ?? []) {
+      for (const run of score.judgeRuns ?? []) {
+        runUsages.push(run.usage);
+        addRunUsage(usage2, run.usage);
+      }
+    }
   }
+  omitPartialCost(usage2, runUsages);
   return usage2;
 }
+function omitPartialCost(total, usages) {
+  if (usages.some(hasUsageWithoutCost)) {
+    total.costUsd = void 0;
+  }
+}
+function hasUsageWithoutCost(usage2) {
+  return usage2?.costUsd === void 0 && ((usage2?.totalTokens ?? (usage2?.inputTokens ?? 0) + (usage2?.outputTokens ?? 0) + (usage2?.reasoningTokens ?? 0)) > 0 || (usage2?.toolCalls ?? 0) > 0);
+}
 function toolCallCount(testCase) {
   const usageToolCalls = testCase.harness?.usage?.toolCalls;
   if (usageToolCalls !== void 0) {
@@ -515,8 +566,8 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate,
-      title: "Eval report hard failure",
-      message: `${formatNumber(nonEvalFailures)} non-eval test failure${nonEvalFailures === 1 ? "" : "s"}; ${counts}`
+      title: "Eval run failed",
+      message: `${formatNumber(nonEvalFailures)} test failure${nonEvalFailures === 1 ? "" : "s"} outside eval cases; ${counts}`
     };
   }
   if (report.totals.evalTotal === 0) {
@@ -525,7 +576,7 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate: null,
-      title: "Eval report hard failure",
+      title: "Eval run failed",
       message: "no eval cases were reported"
     };
   }
@@ -535,8 +586,8 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate,
-      title: "Eval report hard failure",
-      message: `vitest run failed without counted test failures; ${counts}`
+      title: "Eval run failed",
+      message: `Vitest failed without reporting a failed test; ${counts}`
     };
   }
   if (minPassRate !== void 0 && (passRate === null || passRate + Number.EPSILON < minPassRate)) {
@@ -546,7 +597,7 @@ function evaluateEvalGate(report, policy = {}) {
       enforced: true,
       passRate,
       title: `Eval pass rate ${formatPercent(passRate)} \u2014 required ${formatPercent(minPassRate)}`,
-      message: `eval pass rate below floor: ${counts}; required >= ${formatPercent(minPassRate)}`
+      message: `pass rate is below the minimum: ${counts}; minimum ${formatPercent(minPassRate)}`
     };
   }
   if (minScoreAverage !== void 0) {
@@ -557,8 +608,8 @@ function evaluateEvalGate(report, policy = {}) {
         status: "failed",
         enforced: true,
         passRate,
-        title: "Eval score gate failed",
-        message: `no score average available; required avg score >= ${formatScore(minScoreAverage)}`
+        title: "Average score unavailable",
+        message: `no average score was reported; minimum ${formatScore(minScoreAverage)}`
       };
     }
     if (average + Number.EPSILON < minScoreAverage) {
@@ -567,8 +618,8 @@ function evaluateEvalGate(report, policy = {}) {
         status: "failed",
         enforced: true,
         passRate,
-        title: `Avg score ${formatScore(average)} \u2014 required ${formatScore(minScoreAverage)}`,
-        message: `avg score below floor: ${counts}; required avg score >= ${formatScore(minScoreAverage)}`
+        title: `Average score ${formatScore(average)} \u2014 required ${formatScore(minScoreAverage)}`,
+        message: `average score is below the minimum: ${counts}; minimum ${formatScore(minScoreAverage)}`
       };
     }
   }
@@ -578,7 +629,7 @@ function evaluateEvalGate(report, policy = {}) {
     enforced: true,
     passRate,
     title: enforcedPassTitle(report, passRate, minPassRate, minScoreAverage),
-    message: `eval gate passed: ${counts}${formatFloorSuffix(minPassRate, minScoreAverage)}`
+    message: `requirements met: ${counts}${formatMinimumSuffix(minPassRate, minScoreAverage)}`
   };
 }
 function computePassRate(report) {
@@ -611,7 +662,7 @@ function resolveMinPassRate(policy) {
 }
 function defaultCheckTitle(report) {
   if (report.failures.length === 0 && report.status === "passed") {
-    return "No eval failures";
+    return "No eval cases failed";
   }
   if (report.failures.length === 0) {
     return "Vitest run failed";
@@ -620,10 +671,10 @@ function defaultCheckTitle(report) {
 }
 function enforcedPassTitle(report, passRate, minPassRate, minScoreAverage) {
   if (minPassRate !== void 0) {
-    return `Eval pass rate ${formatPercent(passRate)} \u2014 floor ${formatPercent(minPassRate)}`;
+    return `Eval pass rate ${formatPercent(passRate)} \u2014 minimum ${formatPercent(minPassRate)}`;
   }
   if (minScoreAverage !== void 0) {
-    return `Avg score ${formatScore(report.score?.average)} \u2014 floor ${formatScore(minScoreAverage)}`;
+    return `Average score ${formatScore(report.score?.average)} \u2014 minimum ${formatScore(minScoreAverage)}`;
   }
   return defaultCheckTitle(report);
 }
@@ -632,19 +683,22 @@ function formatEvalCounts(report, passRate) {
   const passRateText = passRate === null ? "n/a" : formatPercent(passRate);
   return `${formatNumber(report.totals.evalPassed)}/${formatNumber(
     report.totals.evalTotal
-  )} passed (${passRateText}), avg score ${scoreText}`;
+  )} passed (${passRateText}), average score ${scoreText}`;
 }
-function formatFloorSuffix(minPassRate, minScoreAverage) {
+function formatMinimumSuffix(minPassRate, minScoreAverage) {
   const parts = [];
   if (minPassRate !== void 0) {
-    parts.push(`pass rate floor ${formatPercent(minPassRate)}`);
+    parts.push(`minimum pass rate ${formatPercent(minPassRate)}`);
   }
   if (minScoreAverage !== void 0) {
-    parts.push(`avg score floor ${formatScore(minScoreAverage)}`);
+    parts.push(`minimum average score ${formatScore(minScoreAverage)}`);
   }
   return parts.length === 0 ? "" : `; ${parts.join(", ")}`;
 }
 
+// src/github.ts
+var import_node_fs = require("fs");
+
 // src/summary.ts
 var DEFAULT_MAX_FAILURES = 20;
 var DEFAULT_MAX_REASON_CHARS = 8e3;
@@ -672,23 +726,23 @@ function renderJobSummary(report, options = {}) {
     ""
   ];
   if (report.failures.length > 0) {
-    const failureHeading = options.gate?.ok === true ? "### Quality Misses" : "### Failures";
+    const failureHeading = options.gate?.ok === true ? "### Cases Below Target" : "### Failures";
     lines.push(failureHeading, "");
     failures.forEach((testCase, index) => {
       lines.push(...renderFailureDetails(testCase, index + 1, options), "");
     });
     if (report.failures.length > failures.length) {
-      const omittedLabel = options.gate?.ok === true ? "quality misses" : "failures";
+      const omittedLabel = options.gate?.ok === true ? "cases below target" : "failures";
       lines.push(
         `${report.failures.length - failures.length} more ${omittedLabel} omitted from this summary.`,
         ""
       );
     }
   } else if (report.totals.evalTotal > 0) {
-    lines.push("### Failures", "", "No eval failures.", "");
+    lines.push("### Failures", "", "No eval cases failed.", "");
   }
   if (report.totals.evalTotal === 0) {
-    lines.push("No eval metadata was found in the Vitest JSON report.", "");
+    lines.push("No eval results were found in the Vitest JSON report.", "");
   }
   return `${lines.join("\n")}
 `;
@@ -698,9 +752,9 @@ function formatCountLine(passed, failed, total) {
 }
 function renderSummaryTable(report, nonEvalFailures, gate) {
   const rows = [
-    ["Status", gate?.status ?? report.status],
+    ["Status", capitalize(gate?.status ?? report.status)],
     [
-      "Evals",
+      "Eval cases",
       formatCountLine(
         report.totals.evalPassed,
         report.totals.evalFailed,
@@ -709,23 +763,31 @@ function renderSummaryTable(report, nonEvalFailures, gate) {
     ]
   ];
   if (gate?.passRate !== void 0 && gate.passRate !== null) {
-    rows.push(["Pass Rate", formatPercent(gate.passRate)]);
+    rows.push(["Pass rate", formatPercent(gate.passRate)]);
   } else if (report.totals.evalTotal > 0) {
     rows.push([
-      "Pass Rate",
+      "Pass rate",
       formatPercent(report.totals.evalPassed / report.totals.evalTotal)
     ]);
   }
   if (report.score) {
     rows.push(["Score", formatScoreSummary(report.score)]);
   }
+  if (hasUsage(report.usage) || hasUsage(report.judgeUsage)) {
+    rows.push(["App usage", formatUsage(report.usage)]);
+    rows.push(["Judge usage", formatUsage(report.judgeUsage)]);
+    rows.push([
+      "Total usage",
+      formatUsage(sumUsage(report.usage, report.judgeUsage))
+    ]);
+  }
   if (gate?.enforced) {
-    rows.push(["Gate", gate.message]);
+    rows.push(["Requirements", gate.message]);
   }
   if (nonEvalFailures > 0) {
     rows.push([
-      "Other Failures",
-      `${formatNumber(nonEvalFailures)} non-eval test failure${nonEvalFailures === 1 ? "" : "s"}`
+      "Other test failures",
+      `${formatNumber(nonEvalFailures)} test failure${nonEvalFailures === 1 ? "" : "s"} outside eval cases`
     ]);
   }
   rows.push(["Duration", formatDuration(report.durationMs)]);
@@ -738,7 +800,53 @@ function renderSummaryTable(report, nonEvalFailures, gate) {
   ];
 }
 function formatScoreSummary(score) {
-  return `avg ${formatScore(score.average)}${score.minimum === void 0 ? "" : `, min ${formatScore(score.minimum)}`}`;
+  return `average ${formatScore(score.average)}${score.minimum === void 0 ? "" : `, lowest ${formatScore(score.minimum)}`}`;
+}
+function capitalize(value) {
+  return value.charAt(0).toUpperCase() + value.slice(1);
+}
+function hasUsage(usage2) {
+  return usage2.totalTokens > 0 || usage2.costUsd !== void 0 || usage2.toolCalls > 0;
+}
+function sumUsage(app, judge) {
+  const appCostKnown = !hasUsageWithoutCost2(app);
+  const judgeCostKnown = !hasUsageWithoutCost2(judge);
+  return {
+    inputTokens: app.inputTokens + judge.inputTokens,
+    outputTokens: app.outputTokens + judge.outputTokens,
+    reasoningTokens: app.reasoningTokens + judge.reasoningTokens,
+    totalTokens: app.totalTokens + judge.totalTokens,
+    ...appCostKnown && judgeCostKnown ? { costUsd: (app.costUsd ?? 0) + (judge.costUsd ?? 0) } : {},
+    toolCalls: app.toolCalls + judge.toolCalls
+  };
+}
+function hasUsageWithoutCost2(usage2) {
+  return usage2.costUsd === void 0 && (usage2.totalTokens > 0 || usage2.toolCalls > 0);
+}
+function formatUsage(usage2) {
+  const parts = [];
+  if (usage2.totalTokens > 0) {
+    parts.push(`${formatNumber(usage2.totalTokens)} tokens`);
+  }
+  if (usage2.costUsd !== void 0) {
+    parts.push(
+      usage2.costUsd.toLocaleString("en-US", {
+        style: "currency",
+        currency: "USD",
+        minimumFractionDigits: 2,
+        maximumFractionDigits: 6
+      })
+    );
+  }
+  if (hasUsageWithoutCost2(usage2)) {
+    parts.push("cost unavailable");
+  }
+  if (usage2.toolCalls > 0) {
+    parts.push(
+      `${formatNumber(usage2.toolCalls)} tool call${usage2.toolCalls === 1 ? "" : "s"}`
+    );
+  }
+  return parts.join(", ") || "none";
 }
 function escapeTableCell(value) {
   return value.replace(/\r?\n/g, " ").replace(/\\/g, "\\\\").replace(/\|/g, "\\|");
@@ -818,7 +926,7 @@ function renderFailureBlock(testCase, {
     ["Case", `${number}. ${testCase.displayName}`],
     ["Status", testCase.status],
     ["Location", formatLocation(testCase.displayFile, testCase.location)],
-    ["Harness", testCase.harness?.name ?? "n/a"],
+    ["App runner", testCase.harness?.name ?? "n/a"],
     ["Score", formatScore(failure?.score ?? testCase.eval?.avgScore)],
     ["Judge", failure?.judgeName ?? "n/a"]
   ];
@@ -856,7 +964,7 @@ function renderFailureBlock(testCase, {
   if (finalOutput !== void 0) {
     lines.push(
       ...renderAsciiSection(
-        "Final Output",
+        "Output",
         stringifyValue(finalOutput, maxOutputChars).split(/\r?\n/)
       ),
       ""
@@ -884,7 +992,7 @@ function renderFailureBlock(testCase, {
   if (testCase.harness?.errors.length) {
     lines.push(
       ...renderAsciiSection(
-        "Harness Errors",
+        "App errors",
         stringifyValue(testCase.harness.errors, maxReasonChars).split(/\r?\n/)
       ),
       ""
@@ -933,7 +1041,9 @@ function formatCaseUsage(testCase) {
     parts.push(`${formatNumber(totalTokens)} tokens`);
   }
   if (toolCalls > 0) {
-    parts.push(`${formatNumber(toolCalls)} tool${toolCalls === 1 ? "" : "s"}`);
+    parts.push(
+      `${formatNumber(toolCalls)} tool call${toolCalls === 1 ? "" : "s"}`
+    );
   }
   if (testCase.harness?.timingMs !== void 0) {
     parts.push(formatDuration(testCase.harness.timingMs));
@@ -945,10 +1055,44 @@ function formatCaseUsage(testCase) {
 var DEFAULT_CHECK_NAME = "vitest-evals";
 var MAX_CHECK_SUMMARY_LENGTH = 64e3;
 var CHECK_SUMMARY_TRUNCATION_SUFFIX = "\n\n[truncated for GitHub Check Run]\n";
+function resolveCheckSha(env = process.env, options = {}) {
+  const explicit = options.sha?.trim() || env.GITHUB_PR_HEAD_SHA?.trim();
+  if (explicit) {
+    return explicit;
+  }
+  const eventPath = options.eventPath?.trim() || env.GITHUB_EVENT_PATH?.trim();
+  if (eventPath) {
+    try {
+      const event = JSON.parse((0, import_node_fs.readFileSync)(eventPath, "utf8"));
+      const headSha = event.pull_request?.head?.sha;
+      if (typeof headSha === "string" && headSha.trim()) {
+        return headSha.trim();
+      }
+    } catch {
+    }
+  }
+  return env.GITHUB_SHA?.trim() || void 0;
+}
+function resolveCheckDetailsUrl(env = process.env, options = {}) {
+  const explicit = options.detailsUrl?.trim();
+  if (explicit) {
+    return explicit;
+  }
+  const server = env.GITHUB_SERVER_URL?.replace(/\/$/, "");
+  const repository = env.GITHUB_REPOSITORY?.trim();
+  const runId = env.GITHUB_RUN_ID?.trim();
+  if (!server || !repository || !runId) {
+    return void 0;
+  }
+  return `${server}/${repository}/actions/runs/${runId}`;
+}
 async function publishCheckRun(report, options = {}) {
   const token = options.token ?? process.env.GITHUB_TOKEN;
   const repository = options.repository ?? process.env.GITHUB_REPOSITORY;
-  const sha = options.sha ?? process.env.GITHUB_SHA;
+  const sha = resolveCheckSha(process.env, { sha: options.sha });
+  const detailsUrl = resolveCheckDetailsUrl(process.env, {
+    detailsUrl: options.detailsUrl
+  });
   if (!token) {
     return { status: "skipped", reason: "missing GITHUB_TOKEN" };
   }
@@ -956,7 +1100,10 @@ async function publishCheckRun(report, options = {}) {
     return { status: "skipped", reason: "missing GITHUB_REPOSITORY" };
   }
   if (!sha && options.checkRunId === void 0) {
-    return { status: "skipped", reason: "missing GITHUB_SHA" };
+    return {
+      status: "skipped",
+      reason: "missing commit SHA (set --sha / options.sha, GITHUB_PR_HEAD_SHA, pull_request.head.sha, or GITHUB_SHA)"
+    };
   }
   const [owner, repo] = repository.split("/");
   if (!owner || !repo) {
@@ -965,7 +1112,7 @@ async function publishCheckRun(report, options = {}) {
       reason: `invalid GitHub repository: ${repository}`
     };
   }
-  const payload = buildCheckRunPayload(report, options);
+  const payload = buildCheckRunPayload(report, options, detailsUrl);
   const apiUrl = options.apiUrl ?? process.env.GITHUB_API_URL ?? "https://api.github.com";
   const requestUrl = options.checkRunId === void 0 ? `${apiUrl}/repos/${owner}/${repo}/check-runs` : `${apiUrl}/repos/${owner}/${repo}/check-runs/${options.checkRunId}`;
   const response = await fetch(requestUrl, {
@@ -980,6 +1127,7 @@ async function publishCheckRun(report, options = {}) {
       options.checkRunId === void 0 ? {
         name: options.name ?? DEFAULT_CHECK_NAME,
         head_sha: sha,
+        ...options.externalId ? { external_id: options.externalId } : {},
         ...payload
       } : payload
     )
@@ -994,10 +1142,11 @@ async function publishCheckRun(report, options = {}) {
   return {
     status: options.checkRunId === void 0 ? "created" : "updated",
     id: data.id,
-    htmlUrl: data.html_url
+    htmlUrl: data.html_url,
+    sha
   };
 }
-function buildCheckRunPayload(report, options) {
+function buildCheckRunPayload(report, options, detailsUrl) {
   const gate = options.gate ?? evaluateEvalGate(report);
   const annotations = buildCheckAnnotations(report, {
     maxAnnotations: options.maxAnnotations,
@@ -1007,6 +1156,7 @@ function buildCheckRunPayload(report, options) {
     status: "completed",
     conclusion: gate.ok ? "success" : "failure",
     completed_at: (/* @__PURE__ */ new Date()).toISOString(),
+    ...detailsUrl ? { details_url: detailsUrl } : {},
     output: {
       title: gate.title,
       summary: truncateCheckSummary(
@@ -1059,16 +1209,22 @@ function mergeEvalReports(reports) {
       minimum: Math.min(...scoredCases)
     } : void 0,
     usage: mergeUsage(reports.map((report) => report.usage)),
+    judgeUsage: mergeUsage(reports.map((report) => report.judgeUsage)),
     cases,
     failures
   };
 }
 function mergeUsage(usages) {
+  const costs = usages.map((usage2) => usage2.costUsd).filter((cost) => cost !== void 0);
+  const costComplete = !usages.some(
+    (usage2) => usage2.costUsd === void 0 && (usage2.totalTokens > 0 || usage2.toolCalls > 0)
+  );
   return {
     inputTokens: sum(usages, (usage2) => usage2.inputTokens),
     outputTokens: sum(usages, (usage2) => usage2.outputTokens),
     reasoningTokens: sum(usages, (usage2) => usage2.reasoningTokens),
     totalTokens: sum(usages, (usage2) => usage2.totalTokens),
+    ...costComplete && costs.length > 0 ? { costUsd: costs.reduce((total, cost) => total + cost, 0) } : {},
     toolCalls: sum(usages, (usage2) => usage2.toolCalls)
   };
 }
@@ -1160,11 +1316,17 @@ async function publishEvalReport(options) {
         name: options.checkName,
         repository: options.repository,
         sha: options.sha,
+        detailsUrl: options.detailsUrl,
+        externalId: options.externalId,
         token: options.token,
         gate
       });
       if (checkRun.status === "skipped") {
         options.warn?.(`GitHub Check Run skipped: ${checkRun.reason}`);
+      } else if (checkRun.htmlUrl) {
+        console.log(`published check run: ${checkRun.htmlUrl}`);
+      } else if (checkRun.id !== void 0) {
+        console.log(`published check run id: ${checkRun.id}`);
       }
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -1174,14 +1336,21 @@ async function publishEvalReport(options) {
       options.warn?.(message);
     }
   }
+  const gateFailed = gate.enforced && !gate.ok;
+  const wantsSoftFail = options.softFail ?? options.checkRun === true;
+  const softFail = wantsSoftFail && checkRunPublished(checkRun);
+  const shouldFail = gateFailed && !softFail;
   return {
     report,
     resultFiles,
     gate,
-    shouldFail: gate.enforced && !gate.ok,
+    shouldFail,
     checkRun
   };
 }
+function checkRunPublished(checkRun) {
+  return checkRun?.status === "created" || checkRun?.status === "updated";
+}
 
 // src/cli.ts
 main().catch((error) => {
@@ -1206,6 +1375,7 @@ async function main() {
     checkName: options.checkName,
     failOnCheckError: options.failOnCheckError,
     failOnFailures: options.failOnFailures,
+    softFail: options.softFail,
     minPassRate: options.minPassRate,
     minScoreAverage: options.minScoreAverage,
     maxAnnotations: options.maxAnnotations,
@@ -1241,12 +1411,14 @@ function usage() {
     "  --fail-on-failures       Exit non-zero when any eval case failed",
     "  --min-pass-rate <0-1>    Exit non-zero when eval pass rate is below this floor",
     "  --min-score-average <0-1> Exit non-zero when average score is below this floor",
+    "  --soft-fail              Keep exit 0 when a published Check Run owns gate status",
+    "  --no-soft-fail           Always exit non-zero when an enforced gate fails",
     "  --fail-on-check-error    Fail when Check Run publishing fails",
     "  --check-run-id <id>      Update an existing Check Run",
     "  --check-name <name>      Check Run name (default: vitest-evals)",
     "  --token <token>          GitHub token (default: GITHUB_TOKEN)",
     "  --repo <owner/repo>      GitHub repository (default: GITHUB_REPOSITORY)",
-    "  --sha <sha>              Git commit SHA (default: GITHUB_SHA)",
+    "  --sha <sha>              Git commit SHA (default: PR head, then GITHUB_SHA)",
     "  --workspace <path>       Workspace path for relative annotation files",
     "  --max-annotations <n>    Maximum annotations to emit",
     "  --max-failures <n>       Maximum failures to include in details"