@vitest-evals/github-reporter 0.15.0 → 0.16.0

package/dist/index.mjs

@@ -269,9 +269,10 @@ var DEFAULT_MAX_CHECK_ANNOTATIONS = 50;
 var MAX_CHECK_FIELD_LENGTH = 64e3;
 function renderWorkflowCommands(report, options = {}) {
   const maxAnnotations = options.maxAnnotations ?? DEFAULT_MAX_WORKFLOW_ANNOTATIONS;
+  const command = caseAnnotationCommand(options.gate);
   return report.failures.filter(hasAnnotationLocation).slice(0, maxAnnotations).map(
     (testCase) => formatWorkflowCommand({
-      command: "error",
+      command,
       properties: {
         file: testCase.displayFile,
         line: String(testCase.location.line),
@@ -287,11 +288,12 @@ function buildCheckAnnotations(report, options = {}) {
     options.maxAnnotations ?? DEFAULT_MAX_CHECK_ANNOTATIONS,
     DEFAULT_MAX_CHECK_ANNOTATIONS
   );
+  const annotationLevel = caseAnnotationLevel(options.gate);
   return report.failures.filter(hasAnnotationLocation).slice(0, maxAnnotations).map((testCase) => ({
     path: testCase.displayFile,
     start_line: testCase.location.line,
     end_line: testCase.location.line,
-    annotation_level: "failure",
+    annotation_level: annotationLevel,
     title: truncate(
       `${testCase.primaryFailure?.judgeName ?? "vitest-evals"} - ${testCase.displayName}`,
       255
@@ -303,6 +305,12 @@ function buildCheckAnnotations(report, options = {}) {
     raw_details: truncate(formatRawDetails(testCase), MAX_CHECK_FIELD_LENGTH)
   }));
 }
+function caseAnnotationCommand(gate) {
+  return gate?.ok ? "warning" : "error";
+}
+function caseAnnotationLevel(gate) {
+  return gate?.ok ? "warning" : "failure";
+}
 function hasAnnotationLocation(testCase) {
   return Boolean(testCase.location);
 }
@@ -362,6 +370,164 @@ function formatWorkflowCommand({
   return `::${command} ${renderedProperties}::${escapeCommandData(message)}`;
 }
 
+// src/gate.ts
+function evaluateEvalGate(report, policy = {}) {
+  const minPassRate = resolveMinPassRate(policy);
+  const minScoreAverage = policy.minScoreAverage;
+  const enforced = minPassRate !== void 0 || minScoreAverage !== void 0 || policy.failOnFailures === true;
+  const passRate = computePassRate(report);
+  const counts = formatEvalCounts(report, passRate);
+  if (!enforced) {
+    const ok = report.status === "passed";
+    return {
+      ok,
+      status: ok ? "passed" : "failed",
+      enforced: false,
+      passRate,
+      title: defaultCheckTitle(report),
+      message: ok ? `eval report passed: ${counts}` : `eval report failed: ${counts}`
+    };
+  }
+  const nonEvalFailures = Math.max(
+    0,
+    report.totals.failed - report.totals.evalFailed
+  );
+  if (nonEvalFailures > 0) {
+    return {
+      ok: false,
+      status: "failed",
+      enforced: true,
+      passRate,
+      title: "Eval report hard failure",
+      message: `${formatNumber(nonEvalFailures)} non-eval test failure${nonEvalFailures === 1 ? "" : "s"}; ${counts}`
+    };
+  }
+  if (report.totals.evalTotal === 0) {
+    return {
+      ok: false,
+      status: "failed",
+      enforced: true,
+      passRate: null,
+      title: "Eval report hard failure",
+      message: "no eval cases were reported"
+    };
+  }
+  if (report.status === "failed" && report.totals.failed === 0 && report.failures.length === 0) {
+    return {
+      ok: false,
+      status: "failed",
+      enforced: true,
+      passRate,
+      title: "Eval report hard failure",
+      message: `vitest run failed without counted test failures; ${counts}`
+    };
+  }
+  if (minPassRate !== void 0 && (passRate === null || passRate + Number.EPSILON < minPassRate)) {
+    return {
+      ok: false,
+      status: "failed",
+      enforced: true,
+      passRate,
+      title: `Eval pass rate ${formatPercent(passRate)} \u2014 required ${formatPercent(minPassRate)}`,
+      message: `eval pass rate below floor: ${counts}; required >= ${formatPercent(minPassRate)}`
+    };
+  }
+  if (minScoreAverage !== void 0) {
+    const average = report.score?.average;
+    if (average === void 0 || !Number.isFinite(average)) {
+      return {
+        ok: false,
+        status: "failed",
+        enforced: true,
+        passRate,
+        title: "Eval score gate failed",
+        message: `no score average available; required avg score >= ${formatScore(minScoreAverage)}`
+      };
+    }
+    if (average + Number.EPSILON < minScoreAverage) {
+      return {
+        ok: false,
+        status: "failed",
+        enforced: true,
+        passRate,
+        title: `Avg score ${formatScore(average)} \u2014 required ${formatScore(minScoreAverage)}`,
+        message: `avg score below floor: ${counts}; required avg score >= ${formatScore(minScoreAverage)}`
+      };
+    }
+  }
+  return {
+    ok: true,
+    status: "passed",
+    enforced: true,
+    passRate,
+    title: enforcedPassTitle(report, passRate, minPassRate, minScoreAverage),
+    message: `eval gate passed: ${counts}${formatFloorSuffix(minPassRate, minScoreAverage)}`
+  };
+}
+function computePassRate(report) {
+  if (report.totals.evalTotal <= 0) {
+    return null;
+  }
+  return report.totals.evalPassed / report.totals.evalTotal;
+}
+function formatPercent(value) {
+  if (value == null || !Number.isFinite(value)) {
+    return "n/a";
+  }
+  return `${(value * 100).toFixed(1)}%`;
+}
+function renderGateWorkflowCommand(gate) {
+  if (gate.ok || !gate.enforced) {
+    return void 0;
+  }
+  return `::error title=${escapeCommandProperty(gate.title)}::${escapeCommandData(gate.message)}`;
+}
+function resolveMinPassRate(policy) {
+  const configured = policy.minPassRate;
+  if (policy.failOnFailures) {
+    if (configured === void 0) {
+      return 1;
+    }
+    return Math.max(configured, 1);
+  }
+  return configured;
+}
+function defaultCheckTitle(report) {
+  if (report.failures.length === 0 && report.status === "passed") {
+    return "No eval failures";
+  }
+  if (report.failures.length === 0) {
+    return "Vitest run failed";
+  }
+  return `${report.failures.length} eval failure${report.failures.length === 1 ? "" : "s"}`;
+}
+function enforcedPassTitle(report, passRate, minPassRate, minScoreAverage) {
+  if (minPassRate !== void 0) {
+    return `Eval pass rate ${formatPercent(passRate)} \u2014 floor ${formatPercent(minPassRate)}`;
+  }
+  if (minScoreAverage !== void 0) {
+    return `Avg score ${formatScore(report.score?.average)} \u2014 floor ${formatScore(minScoreAverage)}`;
+  }
+  return defaultCheckTitle(report);
+}
+function formatEvalCounts(report, passRate) {
+  const scoreText = report.score?.average === void 0 ? "n/a" : formatScore(report.score.average);
+  const passRateText = passRate === null ? "n/a" : formatPercent(passRate);
+  return `${formatNumber(report.totals.evalPassed)}/${formatNumber(
+    report.totals.evalTotal
+  )} passed (${passRateText}), avg score ${scoreText}`;
+}
+function formatFloorSuffix(minPassRate, minScoreAverage) {
+  const parts = [];
+  if (minPassRate !== void 0) {
+    parts.push(`pass rate floor ${formatPercent(minPassRate)}`);
+  }
+  if (minScoreAverage !== void 0) {
+    parts.push(`avg score floor ${formatScore(minScoreAverage)}`);
+  }
+  return parts.length === 0 ? "" : `; ${parts.join(", ")}`;
+}
+
 // src/summary.ts
 var DEFAULT_MAX_FAILURES = 20;
 var DEFAULT_MAX_REASON_CHARS = 8e3;
@@ -382,20 +548,22 @@ function renderJobSummary(report, options = {}) {
   const lines = [
     "# vitest-evals",
     "",
-    ...renderSummaryTable(report, nonEvalFailures),
+    ...renderSummaryTable(report, nonEvalFailures, options.gate),
     "",
     ...renderScoreDistribution(report),
     "## Results",
     ""
   ];
   if (report.failures.length > 0) {
-    lines.push("### Failures", "");
+    const failureHeading = options.gate?.ok === true ? "### Quality Misses" : "### Failures";
+    lines.push(failureHeading, "");
     failures.forEach((testCase, index) => {
       lines.push(...renderFailureDetails(testCase, index + 1, options), "");
     });
     if (report.failures.length > failures.length) {
+      const omittedLabel = options.gate?.ok === true ? "quality misses" : "failures";
       lines.push(
-        `${report.failures.length - failures.length} more failures omitted from this summary.`,
+        `${report.failures.length - failures.length} more ${omittedLabel} omitted from this summary.`,
         ""
       );
     }
@@ -411,9 +579,9 @@ function renderJobSummary(report, options = {}) {
 function formatCountLine(passed, failed, total) {
   return `${formatNumber(passed)} passed, ${formatNumber(failed)} failed, ${formatNumber(total)} total`;
 }
-function renderSummaryTable(report, nonEvalFailures) {
+function renderSummaryTable(report, nonEvalFailures, gate) {
   const rows = [
-    ["Status", report.status],
+    ["Status", gate?.status ?? report.status],
     [
       "Evals",
       formatCountLine(
@@ -423,9 +591,20 @@ function renderSummaryTable(report, nonEvalFailures) {
       )
     ]
   ];
+  if (gate?.passRate !== void 0 && gate.passRate !== null) {
+    rows.push(["Pass Rate", formatPercent(gate.passRate)]);
+  } else if (report.totals.evalTotal > 0) {
+    rows.push([
+      "Pass Rate",
+      formatPercent(report.totals.evalPassed / report.totals.evalTotal)
+    ]);
+  }
   if (report.score) {
     rows.push(["Score", formatScoreSummary(report.score)]);
   }
+  if (gate?.enforced) {
+    rows.push(["Gate", gate.message]);
+  }
   if (nonEvalFailures > 0) {
     rows.push([
       "Other Failures",
@@ -702,23 +881,25 @@ async function publishCheckRun(report, options = {}) {
   };
 }
 function buildCheckRunPayload(report, options) {
+  const gate = options.gate ?? evaluateEvalGate(report);
   const annotations = buildCheckAnnotations(report, {
-    maxAnnotations: options.maxAnnotations
+    maxAnnotations: options.maxAnnotations,
+    gate
   });
-  const title = report.failures.length === 0 && report.status === "passed" ? "No eval failures" : report.failures.length === 0 ? "Vitest run failed" : `${report.failures.length} eval failure${report.failures.length === 1 ? "" : "s"}`;
   return {
     status: "completed",
-    conclusion: report.status === "passed" ? "success" : "failure",
+    conclusion: gate.ok ? "success" : "failure",
     completed_at: (/* @__PURE__ */ new Date()).toISOString(),
     output: {
-      title,
+      title: gate.title,
       summary: truncateCheckSummary(
         renderJobSummary(report, {
           ...options,
           maxFailures: options.maxFailures ?? 5,
           maxReasonChars: options.maxReasonChars ?? 4e3,
           maxOutputChars: options.maxOutputChars ?? 2e3,
-          maxToolCalls: options.maxToolCalls ?? 10
+          maxToolCalls: options.maxToolCalls ?? 10,
+          gate
         })
       ),
       annotations
@@ -731,10 +912,175 @@ function truncateCheckSummary(summary) {
   }
   return `${summary.slice(0, MAX_CHECK_SUMMARY_LENGTH - CHECK_SUMMARY_TRUNCATION_SUFFIX.length).trimEnd()}${CHECK_SUMMARY_TRUNCATION_SUFFIX}`;
 }
+
+// src/report.ts
+import { appendFile } from "fs/promises";
+import {
+  readVitestJsonReportFile,
+  resolveResultFiles
+} from "@vitest-evals/core/node";
+
+// src/merge.ts
+function mergeEvalReports(reports) {
+  const cases = reports.flatMap((report) => report.cases);
+  const failures = reports.flatMap((report) => report.failures);
+  const scoredCases = cases.map((testCase) => testCase.eval?.avgScore).filter(
+    (score) => typeof score === "number" && Number.isFinite(score)
+  );
+  const startedAtValues = reports.map((report) => report.startedAt).filter(
+    (startedAt2) => typeof startedAt2 === "number" && Number.isFinite(startedAt2)
+  );
+  const startedAt = startedAtValues.length > 0 ? Math.min(...startedAtValues) : void 0;
+  return {
+    status: reports.some((report) => report.status === "failed") ? "failed" : "passed",
+    startedAt,
+    durationMs: mergeDuration(reports),
+    totals: {
+      total: sum(reports, (report) => report.totals.total),
+      passed: sum(reports, (report) => report.totals.passed),
+      failed: sum(reports, (report) => report.totals.failed),
+      skipped: sum(reports, (report) => report.totals.skipped),
+      evalTotal: sum(reports, (report) => report.totals.evalTotal),
+      evalPassed: sum(reports, (report) => report.totals.evalPassed),
+      evalFailed: sum(reports, (report) => report.totals.evalFailed)
+    },
+    score: scoredCases.length > 0 ? {
+      average: scoredCases.reduce((total, score) => total + score, 0) / scoredCases.length,
+      minimum: Math.min(...scoredCases)
+    } : void 0,
+    usage: mergeUsage(reports.map((report) => report.usage)),
+    cases,
+    failures
+  };
+}
+function mergeUsage(usages) {
+  return {
+    inputTokens: sum(usages, (usage) => usage.inputTokens),
+    outputTokens: sum(usages, (usage) => usage.outputTokens),
+    reasoningTokens: sum(usages, (usage) => usage.reasoningTokens),
+    totalTokens: sum(usages, (usage) => usage.totalTokens),
+    toolCalls: sum(usages, (usage) => usage.toolCalls)
+  };
+}
+function mergeDuration(reports) {
+  const durations = reports.map((report) => report.durationMs).filter(
+    (durationMs) => typeof durationMs === "number" && Number.isFinite(durationMs)
+  );
+  const intervals = reports.map((report) => {
+    if (typeof report.startedAt !== "number" || !Number.isFinite(report.startedAt) || typeof report.durationMs !== "number" || !Number.isFinite(report.durationMs)) {
+      return void 0;
+    }
+    return {
+      start: report.startedAt,
+      end: report.startedAt + report.durationMs
+    };
+  }).filter(
+    (interval) => Boolean(interval)
+  );
+  if (intervals.length > 0 && intervals.length === durations.length) {
+    return Math.max(...intervals.map((interval) => interval.end)) - Math.min(...intervals.map((interval) => interval.start));
+  }
+  return durations.length > 0 ? durations.reduce((total, durationMs) => total + durationMs, 0) : void 0;
+}
+function sum(items, select) {
+  return items.reduce((total, item) => total + select(item), 0);
+}
+
+// src/report.ts
+async function publishEvalReport(options) {
+  const resultFiles = await resolveResultFiles(options.resultPatterns, {
+    cwd: options.cwd
+  });
+  if (resultFiles.length === 0) {
+    throw new Error(
+      `No eval result files matched: ${options.resultPatterns.join(", ")}`
+    );
+  }
+  const reports = await Promise.all(
+    resultFiles.map(async (resultFile) => {
+      const json = await readVitestJsonReportFile(resultFile);
+      return collectEvalReport(json, {
+        workspace: options.workspace
+      });
+    })
+  );
+  const report = mergeEvalReports(reports);
+  const gate = evaluateEvalGate(report, {
+    failOnFailures: options.failOnFailures,
+    minPassRate: options.minPassRate,
+    minScoreAverage: options.minScoreAverage
+  });
+  const summary = renderJobSummary(report, {
+    maxFailures: options.maxFailures,
+    maxOutputChars: options.maxOutputChars,
+    maxReasonChars: options.maxReasonChars,
+    maxToolCalls: options.maxToolCalls,
+    gate
+  });
+  if (options.summaryEnabled !== false) {
+    if (options.summaryPath) {
+      await appendFile(options.summaryPath, `${summary}
+`);
+    } else {
+      console.log(summary);
+    }
+  }
+  if (options.annotations) {
+    const gateCommand = renderGateWorkflowCommand(gate);
+    if (gateCommand) {
+      console.log(gateCommand);
+    }
+    for (const command of renderWorkflowCommands(report, {
+      maxAnnotations: options.maxAnnotations,
+      gate
+    })) {
+      console.log(command);
+    }
+  }
+  let checkRun;
+  if (options.checkRun) {
+    try {
+      checkRun = await publishCheckRun(report, {
+        checkRunId: options.checkRunId,
+        maxAnnotations: options.maxAnnotations,
+        maxFailures: options.maxFailures,
+        maxOutputChars: options.maxOutputChars,
+        maxReasonChars: options.maxReasonChars,
+        maxToolCalls: options.maxToolCalls,
+        name: options.checkName,
+        repository: options.repository,
+        sha: options.sha,
+        token: options.token,
+        gate
+      });
+      if (checkRun.status === "skipped") {
+        options.warn?.(`GitHub Check Run skipped: ${checkRun.reason}`);
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      if (options.failOnCheckError) {
+        throw error;
+      }
+      options.warn?.(message);
+    }
+  }
+  return {
+    report,
+    resultFiles,
+    gate,
+    shouldFail: gate.enforced && !gate.ok,
+    checkRun
+  };
+}
 export {
   buildCheckAnnotations,
   collectEvalReport,
+  computePassRate,
+  evaluateEvalGate,
+  formatPercent,
   publishCheckRun,
+  publishEvalReport,
+  renderGateWorkflowCommand,
   renderJobSummary,
   renderWorkflowCommands
 };