@vitest-evals/github-reporter 0.14.0 → 0.16.0

package/README.md

@@ -36,6 +36,24 @@ steps:
 If configuration or permission is missing, the action keeps the job summary and
 workflow annotations and warns instead of failing.
 
+## Score and Pass-Rate Gates
+
+```yaml
+- id: report
+  uses: getsentry/vitest-evals@v0
+  with:
+    results: eval-results/*.json
+    publish-check: true
+    min-pass-rate: 0.8
+```
+
+- `fail-on-failures: true` requires every eval case to pass
+- `min-pass-rate` and `min-score-average` set aggregate floors in the `0`-`1` range
+- `status`, Check Run conclusion/title, and step exit follow the gate
+- quality misses become warnings when the gate still passes
+- non-eval / infrastructure failures still fail hard
+- use `evals-failed` / `pass-rate` for raw tallies (`pass-rate` is a 0-1 ratio)
+
 ## Sharded Reports
 
 Upload one JSON artifact per shard, then publish one combined report from a
@@ -52,6 +70,7 @@ final reducer job:
   with:
     results: eval-results/*.json
     publish-check: true
+    min-pass-rate: 0.8
 ```
 
 ## Inputs
@@ -64,7 +83,9 @@ final reducer job:
 | `publish-check` | `false` | Publish one GitHub Check Run for the combined report. |
 | `check-name` | `vitest-evals` | Name of the GitHub Check Run. |
 | `github-token` | `${{ github.token }}` | Token used for Check Run publishing. |
-| `fail-on-failures` | `false` | Fail the action when the combined report failed. |
+| `fail-on-failures` | `false` | Fail the action when any eval case failed. Equivalent to `min-pass-rate: 1`. |
+| `min-pass-rate` | unset | Minimum fraction of eval cases that must pass (`0`-`1`). |
+| `min-score-average` | unset | Minimum average eval score across scored cases (`0`-`1`). |
 | `max-annotations` | unset | Maximum number of failure annotations to publish. Check Run annotations are capped at 50 by GitHub. |
 | `max-failures` | unset | Maximum number of detailed failures to include in summaries and checks. |
 

package/dist/cli.js

@@ -41,6 +41,12 @@ function parseCliArgs(args, env = process.env) {
       case "--fail-on-check-error":
         options.failOnCheckError = true;
         break;
+      case "--min-pass-rate":
+        options.minPassRate = readRatio(args, ++index, arg);
+        break;
+      case "--min-score-average":
+        options.minScoreAverage = readRatio(args, ++index, arg);
+        break;
       case "--max-annotations":
         options.maxAnnotations = readInteger(args, ++index, arg);
         break;
@@ -100,6 +106,14 @@ function readInteger(args, index, flag) {
   }
   return Number(rawValue);
 }
+function readRatio(args, index, flag) {
+  const rawValue = readValue(args, index, flag);
+  const parsed = Number(rawValue);
+  if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) {
+    throw new Error(`Invalid ratio for ${flag}`);
+  }
+  return parsed;
+}
 
 // src/report.ts
 var import_promises = require("fs/promises");
@@ -188,9 +202,10 @@ var DEFAULT_MAX_CHECK_ANNOTATIONS = 50;
 var MAX_CHECK_FIELD_LENGTH = 64e3;
 function renderWorkflowCommands(report, options = {}) {
   const maxAnnotations = options.maxAnnotations ?? DEFAULT_MAX_WORKFLOW_ANNOTATIONS;
+  const command = caseAnnotationCommand(options.gate);
   return report.failures.filter(hasAnnotationLocation).slice(0, maxAnnotations).map(
     (testCase) => formatWorkflowCommand({
-      command: "error",
+      command,
       properties: {
         file: testCase.displayFile,
         line: String(testCase.location.line),
@@ -206,11 +221,12 @@ function buildCheckAnnotations(report, options = {}) {
     options.maxAnnotations ?? DEFAULT_MAX_CHECK_ANNOTATIONS,
     DEFAULT_MAX_CHECK_ANNOTATIONS
   );
+  const annotationLevel = caseAnnotationLevel(options.gate);
   return report.failures.filter(hasAnnotationLocation).slice(0, maxAnnotations).map((testCase) => ({
     path: testCase.displayFile,
     start_line: testCase.location.line,
     end_line: testCase.location.line,
-    annotation_level: "failure",
+    annotation_level: annotationLevel,
     title: truncate(
       `${testCase.primaryFailure?.judgeName ?? "vitest-evals"} - ${testCase.displayName}`,
       255
@@ -222,6 +238,12 @@ function buildCheckAnnotations(report, options = {}) {
     raw_details: truncate(formatRawDetails(testCase), MAX_CHECK_FIELD_LENGTH)
   }));
 }
+function caseAnnotationCommand(gate) {
+  return gate?.ok ? "warning" : "error";
+}
+function caseAnnotationLevel(gate) {
+  return gate?.ok ? "warning" : "failure";
+}
 function hasAnnotationLocation(testCase) {
   return Boolean(testCase.location);
 }
@@ -465,6 +487,164 @@ function toolCallCount(testCase) {
   return testCase.toolCalls.length;
 }
 
+// src/gate.ts
+function evaluateEvalGate(report, policy = {}) {
+  const minPassRate = resolveMinPassRate(policy);
+  const minScoreAverage = policy.minScoreAverage;
+  const enforced = minPassRate !== void 0 || minScoreAverage !== void 0 || policy.failOnFailures === true;
+  const passRate = computePassRate(report);
+  const counts = formatEvalCounts(report, passRate);
+  if (!enforced) {
+    const ok = report.status === "passed";
+    return {
+      ok,
+      status: ok ? "passed" : "failed",
+      enforced: false,
+      passRate,
+      title: defaultCheckTitle(report),
+      message: ok ? `eval report passed: ${counts}` : `eval report failed: ${counts}`
+    };
+  }
+  const nonEvalFailures = Math.max(
+    0,
+    report.totals.failed - report.totals.evalFailed
+  );
+  if (nonEvalFailures > 0) {
+    return {
+      ok: false,
+      status: "failed",
+      enforced: true,
+      passRate,
+      title: "Eval report hard failure",
+      message: `${formatNumber(nonEvalFailures)} non-eval test failure${nonEvalFailures === 1 ? "" : "s"}; ${counts}`
+    };
+  }
+  if (report.totals.evalTotal === 0) {
+    return {
+      ok: false,
+      status: "failed",
+      enforced: true,
+      passRate: null,
+      title: "Eval report hard failure",
+      message: "no eval cases were reported"
+    };
+  }
+  if (report.status === "failed" && report.totals.failed === 0 && report.failures.length === 0) {
+    return {
+      ok: false,
+      status: "failed",
+      enforced: true,
+      passRate,
+      title: "Eval report hard failure",
+      message: `vitest run failed without counted test failures; ${counts}`
+    };
+  }
+  if (minPassRate !== void 0 && (passRate === null || passRate + Number.EPSILON < minPassRate)) {
+    return {
+      ok: false,
+      status: "failed",
+      enforced: true,
+      passRate,
+      title: `Eval pass rate ${formatPercent(passRate)} \u2014 required ${formatPercent(minPassRate)}`,
+      message: `eval pass rate below floor: ${counts}; required >= ${formatPercent(minPassRate)}`
+    };
+  }
+  if (minScoreAverage !== void 0) {
+    const average = report.score?.average;
+    if (average === void 0 || !Number.isFinite(average)) {
+      return {
+        ok: false,
+        status: "failed",
+        enforced: true,
+        passRate,
+        title: "Eval score gate failed",
+        message: `no score average available; required avg score >= ${formatScore(minScoreAverage)}`
+      };
+    }
+    if (average + Number.EPSILON < minScoreAverage) {
+      return {
+        ok: false,
+        status: "failed",
+        enforced: true,
+        passRate,
+        title: `Avg score ${formatScore(average)} \u2014 required ${formatScore(minScoreAverage)}`,
+        message: `avg score below floor: ${counts}; required avg score >= ${formatScore(minScoreAverage)}`
+      };
+    }
+  }
+  return {
+    ok: true,
+    status: "passed",
+    enforced: true,
+    passRate,
+    title: enforcedPassTitle(report, passRate, minPassRate, minScoreAverage),
+    message: `eval gate passed: ${counts}${formatFloorSuffix(minPassRate, minScoreAverage)}`
+  };
+}
+function computePassRate(report) {
+  if (report.totals.evalTotal <= 0) {
+    return null;
+  }
+  return report.totals.evalPassed / report.totals.evalTotal;
+}
+function formatPercent(value) {
+  if (value == null || !Number.isFinite(value)) {
+    return "n/a";
+  }
+  return `${(value * 100).toFixed(1)}%`;
+}
+function renderGateWorkflowCommand(gate) {
+  if (gate.ok || !gate.enforced) {
+    return void 0;
+  }
+  return `::error title=${escapeCommandProperty(gate.title)}::${escapeCommandData(gate.message)}`;
+}
+function resolveMinPassRate(policy) {
+  const configured = policy.minPassRate;
+  if (policy.failOnFailures) {
+    if (configured === void 0) {
+      return 1;
+    }
+    return Math.max(configured, 1);
+  }
+  return configured;
+}
+function defaultCheckTitle(report) {
+  if (report.failures.length === 0 && report.status === "passed") {
+    return "No eval failures";
+  }
+  if (report.failures.length === 0) {
+    return "Vitest run failed";
+  }
+  return `${report.failures.length} eval failure${report.failures.length === 1 ? "" : "s"}`;
+}
+function enforcedPassTitle(report, passRate, minPassRate, minScoreAverage) {
+  if (minPassRate !== void 0) {
+    return `Eval pass rate ${formatPercent(passRate)} \u2014 floor ${formatPercent(minPassRate)}`;
+  }
+  if (minScoreAverage !== void 0) {
+    return `Avg score ${formatScore(report.score?.average)} \u2014 floor ${formatScore(minScoreAverage)}`;
+  }
+  return defaultCheckTitle(report);
+}
+function formatEvalCounts(report, passRate) {
+  const scoreText = report.score?.average === void 0 ? "n/a" : formatScore(report.score.average);
+  const passRateText = passRate === null ? "n/a" : formatPercent(passRate);
+  return `${formatNumber(report.totals.evalPassed)}/${formatNumber(
+    report.totals.evalTotal
+  )} passed (${passRateText}), avg score ${scoreText}`;
+}
+function formatFloorSuffix(minPassRate, minScoreAverage) {
+  const parts = [];
+  if (minPassRate !== void 0) {
+    parts.push(`pass rate floor ${formatPercent(minPassRate)}`);
+  }
+  if (minScoreAverage !== void 0) {
+    parts.push(`avg score floor ${formatScore(minScoreAverage)}`);
+  }
+  return parts.length === 0 ? "" : `; ${parts.join(", ")}`;
+}
+
 // src/summary.ts
 var DEFAULT_MAX_FAILURES = 20;
 var DEFAULT_MAX_REASON_CHARS = 8e3;
@@ -485,20 +665,22 @@ function renderJobSummary(report, options = {}) {
   const lines = [
     "# vitest-evals",
     "",
-    ...renderSummaryTable(report, nonEvalFailures),
+    ...renderSummaryTable(report, nonEvalFailures, options.gate),
     "",
     ...renderScoreDistribution(report),
     "## Results",
     ""
   ];
   if (report.failures.length > 0) {
-    lines.push("### Failures", "");
+    const failureHeading = options.gate?.ok === true ? "### Quality Misses" : "### Failures";
+    lines.push(failureHeading, "");
     failures.forEach((testCase, index) => {
       lines.push(...renderFailureDetails(testCase, index + 1, options), "");
     });
     if (report.failures.length > failures.length) {
+      const omittedLabel = options.gate?.ok === true ? "quality misses" : "failures";
       lines.push(
-        `${report.failures.length - failures.length} more failures omitted from this summary.`,
+        `${report.failures.length - failures.length} more ${omittedLabel} omitted from this summary.`,
         ""
       );
     }
@@ -514,9 +696,9 @@ function renderJobSummary(report, options = {}) {
 function formatCountLine(passed, failed, total) {
   return `${formatNumber(passed)} passed, ${formatNumber(failed)} failed, ${formatNumber(total)} total`;
 }
-function renderSummaryTable(report, nonEvalFailures) {
+function renderSummaryTable(report, nonEvalFailures, gate) {
   const rows = [
-    ["Status", report.status],
+    ["Status", gate?.status ?? report.status],
     [
       "Evals",
       formatCountLine(
@@ -526,9 +708,20 @@ function renderSummaryTable(report, nonEvalFailures) {
       )
     ]
   ];
+  if (gate?.passRate !== void 0 && gate.passRate !== null) {
+    rows.push(["Pass Rate", formatPercent(gate.passRate)]);
+  } else if (report.totals.evalTotal > 0) {
+    rows.push([
+      "Pass Rate",
+      formatPercent(report.totals.evalPassed / report.totals.evalTotal)
+    ]);
+  }
   if (report.score) {
     rows.push(["Score", formatScoreSummary(report.score)]);
   }
+  if (gate?.enforced) {
+    rows.push(["Gate", gate.message]);
+  }
   if (nonEvalFailures > 0) {
     rows.push([
       "Other Failures",
@@ -805,23 +998,25 @@ async function publishCheckRun(report, options = {}) {
   };
 }
 function buildCheckRunPayload(report, options) {
+  const gate = options.gate ?? evaluateEvalGate(report);
   const annotations = buildCheckAnnotations(report, {
-    maxAnnotations: options.maxAnnotations
+    maxAnnotations: options.maxAnnotations,
+    gate
   });
-  const title = report.failures.length === 0 && report.status === "passed" ? "No eval failures" : report.failures.length === 0 ? "Vitest run failed" : `${report.failures.length} eval failure${report.failures.length === 1 ? "" : "s"}`;
   return {
     status: "completed",
-    conclusion: report.status === "passed" ? "success" : "failure",
+    conclusion: gate.ok ? "success" : "failure",
     completed_at: (/* @__PURE__ */ new Date()).toISOString(),
     output: {
-      title,
+      title: gate.title,
       summary: truncateCheckSummary(
         renderJobSummary(report, {
           ...options,
           maxFailures: options.maxFailures ?? 5,
           maxReasonChars: options.maxReasonChars ?? 4e3,
           maxOutputChars: options.maxOutputChars ?? 2e3,
-          maxToolCalls: options.maxToolCalls ?? 10
+          maxToolCalls: options.maxToolCalls ?? 10,
+          gate
         })
       ),
       annotations
@@ -920,11 +1115,17 @@ async function publishEvalReport(options) {
     })
   );
   const report = mergeEvalReports(reports);
+  const gate = evaluateEvalGate(report, {
+    failOnFailures: options.failOnFailures,
+    minPassRate: options.minPassRate,
+    minScoreAverage: options.minScoreAverage
+  });
   const summary = renderJobSummary(report, {
     maxFailures: options.maxFailures,
     maxOutputChars: options.maxOutputChars,
     maxReasonChars: options.maxReasonChars,
-    maxToolCalls: options.maxToolCalls
+    maxToolCalls: options.maxToolCalls,
+    gate
   });
   if (options.summaryEnabled !== false) {
     if (options.summaryPath) {
@@ -935,8 +1136,13 @@ async function publishEvalReport(options) {
     }
   }
   if (options.annotations) {
+    const gateCommand = renderGateWorkflowCommand(gate);
+    if (gateCommand) {
+      console.log(gateCommand);
+    }
     for (const command of renderWorkflowCommands(report, {
-      maxAnnotations: options.maxAnnotations
+      maxAnnotations: options.maxAnnotations,
+      gate
     })) {
       console.log(command);
     }
@@ -954,7 +1160,8 @@ async function publishEvalReport(options) {
         name: options.checkName,
         repository: options.repository,
         sha: options.sha,
-        token: options.token
+        token: options.token,
+        gate
       });
       if (checkRun.status === "skipped") {
         options.warn?.(`GitHub Check Run skipped: ${checkRun.reason}`);
@@ -970,6 +1177,8 @@ async function publishEvalReport(options) {
   return {
     report,
     resultFiles,
+    gate,
+    shouldFail: gate.enforced && !gate.ok,
     checkRun
   };
 }
@@ -996,6 +1205,9 @@ async function main() {
     checkRunId: options.checkRunId,
     checkName: options.checkName,
     failOnCheckError: options.failOnCheckError,
+    failOnFailures: options.failOnFailures,
+    minPassRate: options.minPassRate,
+    minScoreAverage: options.minScoreAverage,
     maxAnnotations: options.maxAnnotations,
     maxFailures: options.maxFailures,
     repository: options.repository,
@@ -1003,7 +1215,8 @@ async function main() {
     token: options.token,
     warn
   });
-  if (options.failOnFailures && result.report.status === "failed") {
+  if (result.shouldFail) {
+    console.error(result.gate.message);
     process.exitCode = 1;
   }
 }
@@ -1025,7 +1238,9 @@ function usage() {
     "  --annotations            Emit GitHub workflow-command annotations",
     "  --no-annotations         Disable workflow-command annotations",
     "  --check-run              Publish a GitHub Check Run when configured",
-    "  --fail-on-failures       Exit non-zero when the combined report failed",
+    "  --fail-on-failures       Exit non-zero when any eval case failed",
+    "  --min-pass-rate <0-1>    Exit non-zero when eval pass rate is below this floor",
+    "  --min-score-average <0-1> Exit non-zero when average score is below this floor",
     "  --fail-on-check-error    Fail when Check Run publishing fails",
     "  --check-run-id <id>      Update an existing Check Run",
     "  --check-name <name>      Check Run name (default: vitest-evals)",