@vitest-evals/github-reporter 0.15.0 → 0.16.1

package/README.md

@@ -31,10 +31,39 @@ steps:
     with:
       results: vitest-results.json
       publish-check: true
+      min-pass-rate: 0.8
 ```
 
+On `pull_request`, the Check Run attaches to the PR head SHA (not the temporary
+merge `GITHUB_SHA`) so it shows on the PR checks list. Override with the `sha`
+input when needed.
+
+When a Check Run publishes successfully with a gate, the action step soft-fails
+by default so the Check Run owns green/red. If publishing is skipped, the step
+still fails on a rejected gate. Set `soft-fail: false` to also fail the job.
+
 If configuration or permission is missing, the action keeps the job summary and
-workflow annotations and warns instead of failing.
+workflow annotations and warns instead of failing solely for the missing check.
+
+## Score and Pass-Rate Gates
+
+```yaml
+- id: report
+  uses: getsentry/vitest-evals@v0
+  with:
+    results: eval-results/*.json
+    publish-check: true
+    min-pass-rate: 0.8
+```
+
+- `fail-on-failures: true` requires every eval case to pass
+- `min-pass-rate` and `min-score-average` set aggregate floors in the `0`-`1` range
+- `status` and Check Run conclusion/title follow the gate
+- quality misses become warnings when the gate still passes
+- non-eval / infrastructure failures still fail hard
+- published Check Runs soft-fail the step by default; set `soft-fail: false` to
+  also fail the workflow job
+- use `evals-failed` / `pass-rate` for raw tallies (`pass-rate` is a 0-1 ratio)
 
 ## Sharded Reports
 
@@ -52,6 +81,7 @@ final reducer job:
   with:
     results: eval-results/*.json
     publish-check: true
+    min-pass-rate: 0.8
 ```
 
 ## Inputs
@@ -61,10 +91,14 @@ final reducer job:
 | `results` | `vitest-results.json` | Vitest JSON result files. Supports paths, `*` and `**` globs, and newline-separated entries. |
 | `publish-summary` | `true` | Write a GitHub Actions job summary. |
 | `publish-annotations` | `true` | Emit GitHub workflow annotations for failed evals. |
-| `publish-check` | `false` | Publish one GitHub Check Run for the combined report. |
+| `publish-check` | `false` | Publish one GitHub Check Run for the combined report. Attaches to PR head on `pull_request`. |
 | `check-name` | `vitest-evals` | Name of the GitHub Check Run. |
 | `github-token` | `${{ github.token }}` | Token used for Check Run publishing. |
-| `fail-on-failures` | `false` | Fail the action when the combined report failed. |
+| `sha` | PR head, else `GITHUB_SHA` | Commit SHA for the Check Run. |
+| `fail-on-failures` | `false` | Fail the action when any eval case failed. Equivalent to `min-pass-rate: 1`. |
+| `soft-fail` | auto | Keep the step green when a published Check Run owns a failed gate. |
+| `min-pass-rate` | unset | Minimum fraction of eval cases that must pass (`0`-`1`). |
+| `min-score-average` | unset | Minimum average eval score across scored cases (`0`-`1`). |
 | `max-annotations` | unset | Maximum number of failure annotations to publish. Check Run annotations are capped at 50 by GitHub. |
 | `max-failures` | unset | Maximum number of detailed failures to include in summaries and checks. |
 

package/dist/cli.js

@@ -41,6 +41,18 @@ function parseCliArgs(args, env = process.env) {
       case "--fail-on-check-error":
         options.failOnCheckError = true;
         break;
+      case "--soft-fail":
+        options.softFail = true;
+        break;
+      case "--no-soft-fail":
+        options.softFail = false;
+        break;
+      case "--min-pass-rate":
+        options.minPassRate = readRatio(args, ++index, arg);
+        break;
+      case "--min-score-average":
+        options.minScoreAverage = readRatio(args, ++index, arg);
+        break;
       case "--max-annotations":
         options.maxAnnotations = readInteger(args, ++index, arg);
         break;
@@ -100,6 +112,14 @@ function readInteger(args, index, flag) {
   }
   return Number(rawValue);
 }
+function readRatio(args, index, flag) {
+  const rawValue = readValue(args, index, flag);
+  const parsed = Number(rawValue);
+  if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) {
+    throw new Error(`Invalid ratio for ${flag}`);
+  }
+  return parsed;
+}
 
 // src/report.ts
 var import_promises = require("fs/promises");
@@ -188,9 +208,10 @@ var DEFAULT_MAX_CHECK_ANNOTATIONS = 50;
 var MAX_CHECK_FIELD_LENGTH = 64e3;
 function renderWorkflowCommands(report, options = {}) {
   const maxAnnotations = options.maxAnnotations ?? DEFAULT_MAX_WORKFLOW_ANNOTATIONS;
+  const command = caseAnnotationCommand(options.gate);
   return report.failures.filter(hasAnnotationLocation).slice(0, maxAnnotations).map(
     (testCase) => formatWorkflowCommand({
-      command: "error",
+      command,
       properties: {
         file: testCase.displayFile,
         line: String(testCase.location.line),
@@ -206,11 +227,12 @@ function buildCheckAnnotations(report, options = {}) {
     options.maxAnnotations ?? DEFAULT_MAX_CHECK_ANNOTATIONS,
     DEFAULT_MAX_CHECK_ANNOTATIONS
   );
+  const annotationLevel = caseAnnotationLevel(options.gate);
   return report.failures.filter(hasAnnotationLocation).slice(0, maxAnnotations).map((testCase) => ({
     path: testCase.displayFile,
     start_line: testCase.location.line,
     end_line: testCase.location.line,
-    annotation_level: "failure",
+    annotation_level: annotationLevel,
     title: truncate(
       `${testCase.primaryFailure?.judgeName ?? "vitest-evals"} - ${testCase.displayName}`,
       255
@@ -222,6 +244,12 @@ function buildCheckAnnotations(report, options = {}) {
     raw_details: truncate(formatRawDetails(testCase), MAX_CHECK_FIELD_LENGTH)
   }));
 }
+function caseAnnotationCommand(gate) {
+  return gate?.ok ? "warning" : "error";
+}
+function caseAnnotationLevel(gate) {
+  return gate?.ok ? "warning" : "failure";
+}
 function hasAnnotationLocation(testCase) {
   return Boolean(testCase.location);
 }
@@ -465,6 +493,167 @@ function toolCallCount(testCase) {
   return testCase.toolCalls.length;
 }
 
+// src/gate.ts
+function evaluateEvalGate(report, policy = {}) {
+  const minPassRate = resolveMinPassRate(policy);
+  const minScoreAverage = policy.minScoreAverage;
+  const enforced = minPassRate !== void 0 || minScoreAverage !== void 0 || policy.failOnFailures === true;
+  const passRate = computePassRate(report);
+  const counts = formatEvalCounts(report, passRate);
+  if (!enforced) {
+    const ok = report.status === "passed";
+    return {
+      ok,
+      status: ok ? "passed" : "failed",
+      enforced: false,
+      passRate,
+      title: defaultCheckTitle(report),
+      message: ok ? `eval report passed: ${counts}` : `eval report failed: ${counts}`
+    };
+  }
+  const nonEvalFailures = Math.max(
+    0,
+    report.totals.failed - report.totals.evalFailed
+  );
+  if (nonEvalFailures > 0) {
+    return {
+      ok: false,
+      status: "failed",
+      enforced: true,
+      passRate,
+      title: "Eval report hard failure",
+      message: `${formatNumber(nonEvalFailures)} non-eval test failure${nonEvalFailures === 1 ? "" : "s"}; ${counts}`
+    };
+  }
+  if (report.totals.evalTotal === 0) {
+    return {
+      ok: false,
+      status: "failed",
+      enforced: true,
+      passRate: null,
+      title: "Eval report hard failure",
+      message: "no eval cases were reported"
+    };
+  }
+  if (report.status === "failed" && report.totals.failed === 0 && report.failures.length === 0) {
+    return {
+      ok: false,
+      status: "failed",
+      enforced: true,
+      passRate,
+      title: "Eval report hard failure",
+      message: `vitest run failed without counted test failures; ${counts}`
+    };
+  }
+  if (minPassRate !== void 0 && (passRate === null || passRate + Number.EPSILON < minPassRate)) {
+    return {
+      ok: false,
+      status: "failed",
+      enforced: true,
+      passRate,
+      title: `Eval pass rate ${formatPercent(passRate)} \u2014 required ${formatPercent(minPassRate)}`,
+      message: `eval pass rate below floor: ${counts}; required >= ${formatPercent(minPassRate)}`
+    };
+  }
+  if (minScoreAverage !== void 0) {
+    const average = report.score?.average;
+    if (average === void 0 || !Number.isFinite(average)) {
+      return {
+        ok: false,
+        status: "failed",
+        enforced: true,
+        passRate,
+        title: "Eval score gate failed",
+        message: `no score average available; required avg score >= ${formatScore(minScoreAverage)}`
+      };
+    }
+    if (average + Number.EPSILON < minScoreAverage) {
+      return {
+        ok: false,
+        status: "failed",
+        enforced: true,
+        passRate,
+        title: `Avg score ${formatScore(average)} \u2014 required ${formatScore(minScoreAverage)}`,
+        message: `avg score below floor: ${counts}; required avg score >= ${formatScore(minScoreAverage)}`
+      };
+    }
+  }
+  return {
+    ok: true,
+    status: "passed",
+    enforced: true,
+    passRate,
+    title: enforcedPassTitle(report, passRate, minPassRate, minScoreAverage),
+    message: `eval gate passed: ${counts}${formatFloorSuffix(minPassRate, minScoreAverage)}`
+  };
+}
+function computePassRate(report) {
+  if (report.totals.evalTotal <= 0) {
+    return null;
+  }
+  return report.totals.evalPassed / report.totals.evalTotal;
+}
+function formatPercent(value) {
+  if (value == null || !Number.isFinite(value)) {
+    return "n/a";
+  }
+  return `${(value * 100).toFixed(1)}%`;
+}
+function renderGateWorkflowCommand(gate) {
+  if (gate.ok || !gate.enforced) {
+    return void 0;
+  }
+  return `::error title=${escapeCommandProperty(gate.title)}::${escapeCommandData(gate.message)}`;
+}
+function resolveMinPassRate(policy) {
+  const configured = policy.minPassRate;
+  if (policy.failOnFailures) {
+    if (configured === void 0) {
+      return 1;
+    }
+    return Math.max(configured, 1);
+  }
+  return configured;
+}
+function defaultCheckTitle(report) {
+  if (report.failures.length === 0 && report.status === "passed") {
+    return "No eval failures";
+  }
+  if (report.failures.length === 0) {
+    return "Vitest run failed";
+  }
+  return `${report.failures.length} eval failure${report.failures.length === 1 ? "" : "s"}`;
+}
+function enforcedPassTitle(report, passRate, minPassRate, minScoreAverage) {
+  if (minPassRate !== void 0) {
+    return `Eval pass rate ${formatPercent(passRate)} \u2014 floor ${formatPercent(minPassRate)}`;
+  }
+  if (minScoreAverage !== void 0) {
+    return `Avg score ${formatScore(report.score?.average)} \u2014 floor ${formatScore(minScoreAverage)}`;
+  }
+  return defaultCheckTitle(report);
+}
+function formatEvalCounts(report, passRate) {
+  const scoreText = report.score?.average === void 0 ? "n/a" : formatScore(report.score.average);
+  const passRateText = passRate === null ? "n/a" : formatPercent(passRate);
+  return `${formatNumber(report.totals.evalPassed)}/${formatNumber(
+    report.totals.evalTotal
+  )} passed (${passRateText}), avg score ${scoreText}`;
+}
+function formatFloorSuffix(minPassRate, minScoreAverage) {
+  const parts = [];
+  if (minPassRate !== void 0) {
+    parts.push(`pass rate floor ${formatPercent(minPassRate)}`);
+  }
+  if (minScoreAverage !== void 0) {
+    parts.push(`avg score floor ${formatScore(minScoreAverage)}`);
+  }
+  return parts.length === 0 ? "" : `; ${parts.join(", ")}`;
+}
+
+// src/github.ts
+var import_node_fs = require("fs");
+
 // src/summary.ts
 var DEFAULT_MAX_FAILURES = 20;
 var DEFAULT_MAX_REASON_CHARS = 8e3;
@@ -485,20 +674,22 @@ function renderJobSummary(report, options = {}) {
   const lines = [
     "# vitest-evals",
     "",
-    ...renderSummaryTable(report, nonEvalFailures),
+    ...renderSummaryTable(report, nonEvalFailures, options.gate),
     "",
     ...renderScoreDistribution(report),
     "## Results",
     ""
   ];
   if (report.failures.length > 0) {
-    lines.push("### Failures", "");
+    const failureHeading = options.gate?.ok === true ? "### Quality Misses" : "### Failures";
+    lines.push(failureHeading, "");
     failures.forEach((testCase, index) => {
       lines.push(...renderFailureDetails(testCase, index + 1, options), "");
     });
     if (report.failures.length > failures.length) {
+      const omittedLabel = options.gate?.ok === true ? "quality misses" : "failures";
       lines.push(
-        `${report.failures.length - failures.length} more failures omitted from this summary.`,
+        `${report.failures.length - failures.length} more ${omittedLabel} omitted from this summary.`,
         ""
       );
     }
@@ -514,9 +705,9 @@ function renderJobSummary(report, options = {}) {
 function formatCountLine(passed, failed, total) {
   return `${formatNumber(passed)} passed, ${formatNumber(failed)} failed, ${formatNumber(total)} total`;
 }
-function renderSummaryTable(report, nonEvalFailures) {
+function renderSummaryTable(report, nonEvalFailures, gate) {
   const rows = [
-    ["Status", report.status],
+    ["Status", gate?.status ?? report.status],
     [
       "Evals",
       formatCountLine(
@@ -526,9 +717,20 @@ function renderSummaryTable(report, nonEvalFailures) {
       )
     ]
   ];
+  if (gate?.passRate !== void 0 && gate.passRate !== null) {
+    rows.push(["Pass Rate", formatPercent(gate.passRate)]);
+  } else if (report.totals.evalTotal > 0) {
+    rows.push([
+      "Pass Rate",
+      formatPercent(report.totals.evalPassed / report.totals.evalTotal)
+    ]);
+  }
   if (report.score) {
     rows.push(["Score", formatScoreSummary(report.score)]);
   }
+  if (gate?.enforced) {
+    rows.push(["Gate", gate.message]);
+  }
   if (nonEvalFailures > 0) {
     rows.push([
       "Other Failures",
@@ -752,10 +954,44 @@ function formatCaseUsage(testCase) {
 var DEFAULT_CHECK_NAME = "vitest-evals";
 var MAX_CHECK_SUMMARY_LENGTH = 64e3;
 var CHECK_SUMMARY_TRUNCATION_SUFFIX = "\n\n[truncated for GitHub Check Run]\n";
+function resolveCheckSha(env = process.env, options = {}) {
+  const explicit = options.sha?.trim() || env.GITHUB_PR_HEAD_SHA?.trim();
+  if (explicit) {
+    return explicit;
+  }
+  const eventPath = options.eventPath?.trim() || env.GITHUB_EVENT_PATH?.trim();
+  if (eventPath) {
+    try {
+      const event = JSON.parse((0, import_node_fs.readFileSync)(eventPath, "utf8"));
+      const headSha = event.pull_request?.head?.sha;
+      if (typeof headSha === "string" && headSha.trim()) {
+        return headSha.trim();
+      }
+    } catch {
+    }
+  }
+  return env.GITHUB_SHA?.trim() || void 0;
+}
+function resolveCheckDetailsUrl(env = process.env, options = {}) {
+  const explicit = options.detailsUrl?.trim();
+  if (explicit) {
+    return explicit;
+  }
+  const server = env.GITHUB_SERVER_URL?.replace(/\/$/, "");
+  const repository = env.GITHUB_REPOSITORY?.trim();
+  const runId = env.GITHUB_RUN_ID?.trim();
+  if (!server || !repository || !runId) {
+    return void 0;
+  }
+  return `${server}/${repository}/actions/runs/${runId}`;
+}
 async function publishCheckRun(report, options = {}) {
   const token = options.token ?? process.env.GITHUB_TOKEN;
   const repository = options.repository ?? process.env.GITHUB_REPOSITORY;
-  const sha = options.sha ?? process.env.GITHUB_SHA;
+  const sha = resolveCheckSha(process.env, { sha: options.sha });
+  const detailsUrl = resolveCheckDetailsUrl(process.env, {
+    detailsUrl: options.detailsUrl
+  });
   if (!token) {
     return { status: "skipped", reason: "missing GITHUB_TOKEN" };
   }
@@ -763,7 +999,10 @@ async function publishCheckRun(report, options = {}) {
     return { status: "skipped", reason: "missing GITHUB_REPOSITORY" };
   }
   if (!sha && options.checkRunId === void 0) {
-    return { status: "skipped", reason: "missing GITHUB_SHA" };
+    return {
+      status: "skipped",
+      reason: "missing commit SHA (set --sha / options.sha, GITHUB_PR_HEAD_SHA, pull_request.head.sha, or GITHUB_SHA)"
+    };
   }
   const [owner, repo] = repository.split("/");
   if (!owner || !repo) {
@@ -772,7 +1011,7 @@ async function publishCheckRun(report, options = {}) {
       reason: `invalid GitHub repository: ${repository}`
     };
   }
-  const payload = buildCheckRunPayload(report, options);
+  const payload = buildCheckRunPayload(report, options, detailsUrl);
   const apiUrl = options.apiUrl ?? process.env.GITHUB_API_URL ?? "https://api.github.com";
   const requestUrl = options.checkRunId === void 0 ? `${apiUrl}/repos/${owner}/${repo}/check-runs` : `${apiUrl}/repos/${owner}/${repo}/check-runs/${options.checkRunId}`;
   const response = await fetch(requestUrl, {
@@ -787,6 +1026,7 @@ async function publishCheckRun(report, options = {}) {
       options.checkRunId === void 0 ? {
         name: options.name ?? DEFAULT_CHECK_NAME,
         head_sha: sha,
+        ...options.externalId ? { external_id: options.externalId } : {},
         ...payload
       } : payload
     )
@@ -801,27 +1041,31 @@ async function publishCheckRun(report, options = {}) {
   return {
     status: options.checkRunId === void 0 ? "created" : "updated",
     id: data.id,
-    htmlUrl: data.html_url
+    htmlUrl: data.html_url,
+    sha
   };
 }
-function buildCheckRunPayload(report, options) {
+function buildCheckRunPayload(report, options, detailsUrl) {
+  const gate = options.gate ?? evaluateEvalGate(report);
   const annotations = buildCheckAnnotations(report, {
-    maxAnnotations: options.maxAnnotations
+    maxAnnotations: options.maxAnnotations,
+    gate
   });
-  const title = report.failures.length === 0 && report.status === "passed" ? "No eval failures" : report.failures.length === 0 ? "Vitest run failed" : `${report.failures.length} eval failure${report.failures.length === 1 ? "" : "s"}`;
   return {
     status: "completed",
-    conclusion: report.status === "passed" ? "success" : "failure",
+    conclusion: gate.ok ? "success" : "failure",
     completed_at: (/* @__PURE__ */ new Date()).toISOString(),
+    ...detailsUrl ? { details_url: detailsUrl } : {},
     output: {
-      title,
+      title: gate.title,
       summary: truncateCheckSummary(
         renderJobSummary(report, {
           ...options,
           maxFailures: options.maxFailures ?? 5,
           maxReasonChars: options.maxReasonChars ?? 4e3,
           maxOutputChars: options.maxOutputChars ?? 2e3,
-          maxToolCalls: options.maxToolCalls ?? 10
+          maxToolCalls: options.maxToolCalls ?? 10,
+          gate
         })
       ),
       annotations
@@ -920,11 +1164,17 @@ async function publishEvalReport(options) {
     })
   );
   const report = mergeEvalReports(reports);
+  const gate = evaluateEvalGate(report, {
+    failOnFailures: options.failOnFailures,
+    minPassRate: options.minPassRate,
+    minScoreAverage: options.minScoreAverage
+  });
   const summary = renderJobSummary(report, {
     maxFailures: options.maxFailures,
     maxOutputChars: options.maxOutputChars,
     maxReasonChars: options.maxReasonChars,
-    maxToolCalls: options.maxToolCalls
+    maxToolCalls: options.maxToolCalls,
+    gate
   });
   if (options.summaryEnabled !== false) {
     if (options.summaryPath) {
@@ -935,8 +1185,13 @@ async function publishEvalReport(options) {
     }
   }
   if (options.annotations) {
+    const gateCommand = renderGateWorkflowCommand(gate);
+    if (gateCommand) {
+      console.log(gateCommand);
+    }
     for (const command of renderWorkflowCommands(report, {
-      maxAnnotations: options.maxAnnotations
+      maxAnnotations: options.maxAnnotations,
+      gate
     })) {
       console.log(command);
     }
@@ -954,10 +1209,17 @@ async function publishEvalReport(options) {
         name: options.checkName,
         repository: options.repository,
         sha: options.sha,
-        token: options.token
+        detailsUrl: options.detailsUrl,
+        externalId: options.externalId,
+        token: options.token,
+        gate
       });
       if (checkRun.status === "skipped") {
         options.warn?.(`GitHub Check Run skipped: ${checkRun.reason}`);
+      } else if (checkRun.htmlUrl) {
+        console.log(`published check run: ${checkRun.htmlUrl}`);
+      } else if (checkRun.id !== void 0) {
+        console.log(`published check run id: ${checkRun.id}`);
       }
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -967,12 +1229,21 @@ async function publishEvalReport(options) {
       options.warn?.(message);
     }
   }
+  const gateFailed = gate.enforced && !gate.ok;
+  const wantsSoftFail = options.softFail ?? options.checkRun === true;
+  const softFail = wantsSoftFail && checkRunPublished(checkRun);
+  const shouldFail = gateFailed && !softFail;
   return {
     report,
     resultFiles,
+    gate,
+    shouldFail,
     checkRun
   };
 }
+function checkRunPublished(checkRun) {
+  return checkRun?.status === "created" || checkRun?.status === "updated";
+}
 
 // src/cli.ts
 main().catch((error) => {
@@ -996,6 +1267,10 @@ async function main() {
     checkRunId: options.checkRunId,
     checkName: options.checkName,
     failOnCheckError: options.failOnCheckError,
+    failOnFailures: options.failOnFailures,
+    softFail: options.softFail,
+    minPassRate: options.minPassRate,
+    minScoreAverage: options.minScoreAverage,
     maxAnnotations: options.maxAnnotations,
     maxFailures: options.maxFailures,
     repository: options.repository,
@@ -1003,7 +1278,8 @@ async function main() {
     token: options.token,
     warn
   });
-  if (options.failOnFailures && result.report.status === "failed") {
+  if (result.shouldFail) {
+    console.error(result.gate.message);
     process.exitCode = 1;
   }
 }
@@ -1025,13 +1301,17 @@ function usage() {
     "  --annotations            Emit GitHub workflow-command annotations",
     "  --no-annotations         Disable workflow-command annotations",
     "  --check-run              Publish a GitHub Check Run when configured",
-    "  --fail-on-failures       Exit non-zero when the combined report failed",
+    "  --fail-on-failures       Exit non-zero when any eval case failed",
+    "  --min-pass-rate <0-1>    Exit non-zero when eval pass rate is below this floor",
+    "  --min-score-average <0-1> Exit non-zero when average score is below this floor",
+    "  --soft-fail              Keep exit 0 when a published Check Run owns gate status",
+    "  --no-soft-fail           Always exit non-zero when an enforced gate fails",
     "  --fail-on-check-error    Fail when Check Run publishing fails",
     "  --check-run-id <id>      Update an existing Check Run",
     "  --check-name <name>      Check Run name (default: vitest-evals)",
     "  --token <token>          GitHub token (default: GITHUB_TOKEN)",
     "  --repo <owner/repo>      GitHub repository (default: GITHUB_REPOSITORY)",
-    "  --sha <sha>              Git commit SHA (default: GITHUB_SHA)",
+    "  --sha <sha>              Git commit SHA (default: PR head, then GITHUB_SHA)",
     "  --workspace <path>       Workspace path for relative annotation files",
     "  --max-annotations <n>    Maximum annotations to emit",
     "  --max-failures <n>       Maximum failures to include in details"