@vitest-evals/github-reporter 0.16.0 → 0.16.1

package/README.md

@@ -31,10 +31,19 @@ steps:
     with:
       results: vitest-results.json
       publish-check: true
+      min-pass-rate: 0.8
 ```
 
+On `pull_request`, the Check Run attaches to the PR head SHA (not the temporary
+merge `GITHUB_SHA`) so it shows on the PR checks list. Override with the `sha`
+input when needed.
+
+When a Check Run publishes successfully with a gate, the action step soft-fails
+by default so the Check Run owns green/red. If publishing is skipped, the step
+still fails on a rejected gate. Set `soft-fail: false` to also fail the job.
+
 If configuration or permission is missing, the action keeps the job summary and
-workflow annotations and warns instead of failing.
+workflow annotations and warns instead of failing solely for the missing check.
 
 ## Score and Pass-Rate Gates
 
@@ -49,9 +58,11 @@ workflow annotations and warns instead of failing.
 
 - `fail-on-failures: true` requires every eval case to pass
 - `min-pass-rate` and `min-score-average` set aggregate floors in the `0`-`1` range
-- `status`, Check Run conclusion/title, and step exit follow the gate
+- `status` and Check Run conclusion/title follow the gate
 - quality misses become warnings when the gate still passes
 - non-eval / infrastructure failures still fail hard
+- published Check Runs soft-fail the step by default; set `soft-fail: false` to
+  also fail the workflow job
 - use `evals-failed` / `pass-rate` for raw tallies (`pass-rate` is a 0-1 ratio)
 
 ## Sharded Reports
@@ -80,10 +91,12 @@ final reducer job:
 | `results` | `vitest-results.json` | Vitest JSON result files. Supports paths, `*` and `**` globs, and newline-separated entries. |
 | `publish-summary` | `true` | Write a GitHub Actions job summary. |
 | `publish-annotations` | `true` | Emit GitHub workflow annotations for failed evals. |
-| `publish-check` | `false` | Publish one GitHub Check Run for the combined report. |
+| `publish-check` | `false` | Publish one GitHub Check Run for the combined report. Attaches to PR head on `pull_request`. |
 | `check-name` | `vitest-evals` | Name of the GitHub Check Run. |
 | `github-token` | `${{ github.token }}` | Token used for Check Run publishing. |
+| `sha` | PR head, else `GITHUB_SHA` | Commit SHA for the Check Run. |
 | `fail-on-failures` | `false` | Fail the action when any eval case failed. Equivalent to `min-pass-rate: 1`. |
+| `soft-fail` | auto | Keep the step green when a published Check Run owns a failed gate. |
 | `min-pass-rate` | unset | Minimum fraction of eval cases that must pass (`0`-`1`). |
 | `min-score-average` | unset | Minimum average eval score across scored cases (`0`-`1`). |
 | `max-annotations` | unset | Maximum number of failure annotations to publish. Check Run annotations are capped at 50 by GitHub. |

package/dist/cli.js

@@ -41,6 +41,12 @@ function parseCliArgs(args, env = process.env) {
       case "--fail-on-check-error":
         options.failOnCheckError = true;
         break;
+      case "--soft-fail":
+        options.softFail = true;
+        break;
+      case "--no-soft-fail":
+        options.softFail = false;
+        break;
       case "--min-pass-rate":
         options.minPassRate = readRatio(args, ++index, arg);
         break;
@@ -645,6 +651,9 @@ function formatFloorSuffix(minPassRate, minScoreAverage) {
   return parts.length === 0 ? "" : `; ${parts.join(", ")}`;
 }
 
+// src/github.ts
+var import_node_fs = require("fs");
+
 // src/summary.ts
 var DEFAULT_MAX_FAILURES = 20;
 var DEFAULT_MAX_REASON_CHARS = 8e3;
@@ -945,10 +954,44 @@ function formatCaseUsage(testCase) {
 var DEFAULT_CHECK_NAME = "vitest-evals";
 var MAX_CHECK_SUMMARY_LENGTH = 64e3;
 var CHECK_SUMMARY_TRUNCATION_SUFFIX = "\n\n[truncated for GitHub Check Run]\n";
+function resolveCheckSha(env = process.env, options = {}) {
+  const explicit = options.sha?.trim() || env.GITHUB_PR_HEAD_SHA?.trim();
+  if (explicit) {
+    return explicit;
+  }
+  const eventPath = options.eventPath?.trim() || env.GITHUB_EVENT_PATH?.trim();
+  if (eventPath) {
+    try {
+      const event = JSON.parse((0, import_node_fs.readFileSync)(eventPath, "utf8"));
+      const headSha = event.pull_request?.head?.sha;
+      if (typeof headSha === "string" && headSha.trim()) {
+        return headSha.trim();
+      }
+    } catch {
+    }
+  }
+  return env.GITHUB_SHA?.trim() || void 0;
+}
+function resolveCheckDetailsUrl(env = process.env, options = {}) {
+  const explicit = options.detailsUrl?.trim();
+  if (explicit) {
+    return explicit;
+  }
+  const server = env.GITHUB_SERVER_URL?.replace(/\/$/, "");
+  const repository = env.GITHUB_REPOSITORY?.trim();
+  const runId = env.GITHUB_RUN_ID?.trim();
+  if (!server || !repository || !runId) {
+    return void 0;
+  }
+  return `${server}/${repository}/actions/runs/${runId}`;
+}
 async function publishCheckRun(report, options = {}) {
   const token = options.token ?? process.env.GITHUB_TOKEN;
   const repository = options.repository ?? process.env.GITHUB_REPOSITORY;
-  const sha = options.sha ?? process.env.GITHUB_SHA;
+  const sha = resolveCheckSha(process.env, { sha: options.sha });
+  const detailsUrl = resolveCheckDetailsUrl(process.env, {
+    detailsUrl: options.detailsUrl
+  });
   if (!token) {
     return { status: "skipped", reason: "missing GITHUB_TOKEN" };
   }
@@ -956,7 +999,10 @@ async function publishCheckRun(report, options = {}) {
     return { status: "skipped", reason: "missing GITHUB_REPOSITORY" };
   }
   if (!sha && options.checkRunId === void 0) {
-    return { status: "skipped", reason: "missing GITHUB_SHA" };
+    return {
+      status: "skipped",
+      reason: "missing commit SHA (set --sha / options.sha, GITHUB_PR_HEAD_SHA, pull_request.head.sha, or GITHUB_SHA)"
+    };
   }
   const [owner, repo] = repository.split("/");
   if (!owner || !repo) {
@@ -965,7 +1011,7 @@ async function publishCheckRun(report, options = {}) {
       reason: `invalid GitHub repository: ${repository}`
     };
   }
-  const payload = buildCheckRunPayload(report, options);
+  const payload = buildCheckRunPayload(report, options, detailsUrl);
   const apiUrl = options.apiUrl ?? process.env.GITHUB_API_URL ?? "https://api.github.com";
   const requestUrl = options.checkRunId === void 0 ? `${apiUrl}/repos/${owner}/${repo}/check-runs` : `${apiUrl}/repos/${owner}/${repo}/check-runs/${options.checkRunId}`;
   const response = await fetch(requestUrl, {
@@ -980,6 +1026,7 @@ async function publishCheckRun(report, options = {}) {
       options.checkRunId === void 0 ? {
         name: options.name ?? DEFAULT_CHECK_NAME,
         head_sha: sha,
+        ...options.externalId ? { external_id: options.externalId } : {},
         ...payload
       } : payload
     )
@@ -994,10 +1041,11 @@ async function publishCheckRun(report, options = {}) {
   return {
     status: options.checkRunId === void 0 ? "created" : "updated",
     id: data.id,
-    htmlUrl: data.html_url
+    htmlUrl: data.html_url,
+    sha
   };
 }
-function buildCheckRunPayload(report, options) {
+function buildCheckRunPayload(report, options, detailsUrl) {
   const gate = options.gate ?? evaluateEvalGate(report);
   const annotations = buildCheckAnnotations(report, {
     maxAnnotations: options.maxAnnotations,
@@ -1007,6 +1055,7 @@ function buildCheckRunPayload(report, options) {
     status: "completed",
     conclusion: gate.ok ? "success" : "failure",
     completed_at: (/* @__PURE__ */ new Date()).toISOString(),
+    ...detailsUrl ? { details_url: detailsUrl } : {},
     output: {
       title: gate.title,
       summary: truncateCheckSummary(
@@ -1160,11 +1209,17 @@ async function publishEvalReport(options) {
         name: options.checkName,
         repository: options.repository,
         sha: options.sha,
+        detailsUrl: options.detailsUrl,
+        externalId: options.externalId,
         token: options.token,
         gate
       });
       if (checkRun.status === "skipped") {
         options.warn?.(`GitHub Check Run skipped: ${checkRun.reason}`);
+      } else if (checkRun.htmlUrl) {
+        console.log(`published check run: ${checkRun.htmlUrl}`);
+      } else if (checkRun.id !== void 0) {
+        console.log(`published check run id: ${checkRun.id}`);
       }
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -1174,14 +1229,21 @@ async function publishEvalReport(options) {
       options.warn?.(message);
     }
   }
+  const gateFailed = gate.enforced && !gate.ok;
+  const wantsSoftFail = options.softFail ?? options.checkRun === true;
+  const softFail = wantsSoftFail && checkRunPublished(checkRun);
+  const shouldFail = gateFailed && !softFail;
   return {
     report,
     resultFiles,
     gate,
-    shouldFail: gate.enforced && !gate.ok,
+    shouldFail,
     checkRun
   };
 }
+function checkRunPublished(checkRun) {
+  return checkRun?.status === "created" || checkRun?.status === "updated";
+}
 
 // src/cli.ts
 main().catch((error) => {
@@ -1206,6 +1268,7 @@ async function main() {
     checkName: options.checkName,
     failOnCheckError: options.failOnCheckError,
     failOnFailures: options.failOnFailures,
+    softFail: options.softFail,
     minPassRate: options.minPassRate,
     minScoreAverage: options.minScoreAverage,
     maxAnnotations: options.maxAnnotations,
@@ -1241,12 +1304,14 @@ function usage() {
     "  --fail-on-failures       Exit non-zero when any eval case failed",
     "  --min-pass-rate <0-1>    Exit non-zero when eval pass rate is below this floor",
     "  --min-score-average <0-1> Exit non-zero when average score is below this floor",
+    "  --soft-fail              Keep exit 0 when a published Check Run owns gate status",
+    "  --no-soft-fail           Always exit non-zero when an enforced gate fails",
     "  --fail-on-check-error    Fail when Check Run publishing fails",
     "  --check-run-id <id>      Update an existing Check Run",
     "  --check-name <name>      Check Run name (default: vitest-evals)",
     "  --token <token>          GitHub token (default: GITHUB_TOKEN)",
     "  --repo <owner/repo>      GitHub repository (default: GITHUB_REPOSITORY)",
-    "  --sha <sha>              Git commit SHA (default: GITHUB_SHA)",
+    "  --sha <sha>              Git commit SHA (default: PR head, then GITHUB_SHA)",
     "  --workspace <path>       Workspace path for relative annotation files",
     "  --max-annotations <n>    Maximum annotations to emit",
     "  --max-failures <n>       Maximum failures to include in details"