@vitest-evals/github-reporter 0.9.0-beta.5 → 0.9.0

package/dist/cli.js

@@ -1,9 +1,6 @@
 #!/usr/bin/env node
 "use strict";
 
-// src/cli.ts
-var import_promises = require("fs/promises");
-
 // src/cli-options.ts
 function parseCliArgs(args, env = process.env) {
   const options = {
@@ -11,14 +8,16 @@ function parseCliArgs(args, env = process.env) {
     summaryEnabled: true,
     annotations: env.GITHUB_ACTIONS === "true",
     checkRun: false,
+    failOnFailures: false,
     failOnCheckError: false,
+    resultPatterns: [],
     help: false
   };
   for (let index = 0; index < args.length; index += 1) {
     const arg = args[index];
     switch (arg) {
       case "--json":
-        options.jsonPath = readValue(args, ++index, arg);
+        options.resultPatterns.push(readValue(args, ++index, arg));
         break;
       case "--summary":
         options.summaryPath = readValue(args, ++index, arg);
@@ -36,6 +35,9 @@ function parseCliArgs(args, env = process.env) {
       case "--check-run":
         options.checkRun = true;
         break;
+      case "--fail-on-failures":
+        options.failOnFailures = true;
+        break;
       case "--fail-on-check-error":
         options.failOnCheckError = true;
         break;
@@ -69,8 +71,8 @@ function parseCliArgs(args, env = process.env) {
         options.help = true;
         return withDefaultJsonPath(options, env);
       default:
-        if (!arg.startsWith("-") && !options.jsonPath) {
-          options.jsonPath = arg;
+        if (!arg.startsWith("-")) {
+          options.resultPatterns.push(arg);
           break;
         }
         throw new Error(`Unknown argument: ${arg}`);
@@ -81,7 +83,7 @@ function parseCliArgs(args, env = process.env) {
 function withDefaultJsonPath(options, env) {
   return {
     ...options,
-    jsonPath: options.jsonPath || env.VITEST_EVALS_JSON_REPORT || "vitest-results.json"
+    resultPatterns: options.resultPatterns.length > 0 ? options.resultPatterns : [env.VITEST_EVALS_JSON_REPORT || "vitest-results.json"]
   };
 }
 function readValue(args, index, flag) {
@@ -92,13 +94,16 @@ function readValue(args, index, flag) {
   return value;
 }
 function readInteger(args, index, flag) {
-  const value = Number.parseInt(readValue(args, index, flag), 10);
-  if (!Number.isFinite(value) || value < 0) {
+  const rawValue = readValue(args, index, flag);
+  if (!/^\d+$/.test(rawValue)) {
     throw new Error(`Invalid integer for ${flag}`);
   }
-  return value;
+  return Number(rawValue);
 }
 
+// src/report.ts
+var import_promises2 = require("fs/promises");
+
 // src/utils.ts
 var import_node_path = require("path");
 function isRecord(value) {
@@ -190,6 +195,98 @@ function escapeCommandProperty(value) {
   return escapeCommandData(value).replace(/:/g, "%3A").replace(/,/g, "%2C");
 }
 
+// src/annotations.ts
+var DEFAULT_MAX_WORKFLOW_ANNOTATIONS = 10;
+var DEFAULT_MAX_CHECK_ANNOTATIONS = 50;
+var MAX_CHECK_FIELD_LENGTH = 64e3;
+function renderWorkflowCommands(report, options = {}) {
+  const maxAnnotations = options.maxAnnotations ?? DEFAULT_MAX_WORKFLOW_ANNOTATIONS;
+  return report.failures.filter(hasAnnotationLocation).slice(0, maxAnnotations).map(
+    (testCase) => formatWorkflowCommand({
+      command: "error",
+      properties: {
+        file: testCase.displayFile,
+        line: String(testCase.location.line),
+        col: String(testCase.location.column),
+        title: "vitest-evals"
+      },
+      message: formatWorkflowMessage(testCase)
+    })
+  );
+}
+function buildCheckAnnotations(report, options = {}) {
+  const maxAnnotations = Math.min(
+    options.maxAnnotations ?? DEFAULT_MAX_CHECK_ANNOTATIONS,
+    DEFAULT_MAX_CHECK_ANNOTATIONS
+  );
+  return report.failures.filter(hasAnnotationLocation).slice(0, maxAnnotations).map((testCase) => ({
+    path: testCase.displayFile,
+    start_line: testCase.location.line,
+    end_line: testCase.location.line,
+    annotation_level: "failure",
+    title: truncate(
+      `${testCase.primaryFailure?.judgeName ?? "vitest-evals"} - ${testCase.displayName}`,
+      255
+    ),
+    message: truncate(
+      formatWorkflowMessage(testCase),
+      MAX_CHECK_FIELD_LENGTH
+    ),
+    raw_details: truncate(formatRawDetails(testCase), MAX_CHECK_FIELD_LENGTH)
+  }));
+}
+function hasAnnotationLocation(testCase) {
+  return Boolean(testCase.location);
+}
+function formatWorkflowMessage(testCase) {
+  const failure = testCase.primaryFailure;
+  const parts = [
+    testCase.displayName,
+    `score ${formatScore(failure?.score ?? testCase.eval?.avgScore)}`
+  ];
+  if (failure?.judgeName) {
+    parts.push(failure.judgeName);
+  }
+  const reason = compactLine(failure?.reason ?? "", 320);
+  if (reason) {
+    parts.push(reason);
+  }
+  return parts.join(" - ");
+}
+function formatRawDetails(testCase) {
+  const lines = [
+    `Test: ${testCase.displayName}`,
+    `Location: ${testCase.displayFile}:${testCase.location.line}`,
+    `Harness: ${testCase.harness?.name ?? "n/a"}`,
+    `Score: ${formatScore(testCase.primaryFailure?.score ?? testCase.eval?.avgScore)}`,
+    `Judge: ${testCase.primaryFailure?.judgeName ?? "n/a"}`,
+    "",
+    "Reason:",
+    testCase.primaryFailure?.reason ?? "n/a"
+  ];
+  const finalOutput = testCase.eval?.output ?? testCase.harness?.output;
+  if (finalOutput !== void 0) {
+    lines.push("", "Final:", stringifyValue(finalOutput, 8e3));
+  }
+  if (testCase.harness?.toolCalls.length) {
+    lines.push("", "Tools:");
+    for (const toolCall of testCase.harness.toolCalls) {
+      lines.push(
+        `- ${toolCall.name}: ${toolCall.error ? `error: ${toolCall.error}` : "ok"}`
+      );
+    }
+  }
+  return lines.join("\n");
+}
+function formatWorkflowCommand({
+  command,
+  properties,
+  message
+}) {
+  const renderedProperties = Object.entries(properties).map(([key, value]) => `${key}=${escapeCommandProperty(value)}`).join(",");
+  return `::${command} ${renderedProperties}::${escapeCommandData(message)}`;
+}
+
 // src/collect.ts
 function collectEvalReport(input, options = {}) {
   const cases = input.testResults.flatMap(
@@ -310,7 +407,6 @@ function getUsage(value) {
     outputTokens: numberField(value.outputTokens),
     reasoningTokens: numberField(value.reasoningTokens),
     totalTokens: numberField(value.totalTokens),
-    estimatedCost: numberField(value.estimatedCost),
     toolCalls: numberField(value.toolCalls)
   };
 }
@@ -380,7 +476,6 @@ function sumUsage(cases) {
     outputTokens: 0,
     reasoningTokens: 0,
     totalTokens: 0,
-    estimatedCost: 0,
     toolCalls: 0
   };
   for (const testCase of cases) {
@@ -389,7 +484,6 @@ function sumUsage(cases) {
     usage2.outputTokens += caseUsage?.outputTokens ?? 0;
     usage2.reasoningTokens += caseUsage?.reasoningTokens ?? 0;
     usage2.totalTokens += caseUsage?.totalTokens ?? (caseUsage?.inputTokens ?? 0) + (caseUsage?.outputTokens ?? 0) + (caseUsage?.reasoningTokens ?? 0);
-    usage2.estimatedCost += caseUsage?.estimatedCost ?? 0;
     usage2.toolCalls += caseUsage?.toolCalls ?? testCase.harness?.toolCalls.length ?? 0;
   }
   return usage2;
@@ -403,95 +497,6 @@ function resolveRunDuration(input) {
   return Math.max(...endTimes) - Math.min(...startTimes);
 }
 
-// src/annotations.ts
-var DEFAULT_MAX_WORKFLOW_ANNOTATIONS = 10;
-var DEFAULT_MAX_CHECK_ANNOTATIONS = 50;
-var MAX_CHECK_FIELD_LENGTH = 64e3;
-function renderWorkflowCommands(report, options = {}) {
-  const maxAnnotations = options.maxAnnotations ?? DEFAULT_MAX_WORKFLOW_ANNOTATIONS;
-  return report.failures.filter(hasAnnotationLocation).slice(0, maxAnnotations).map(
-    (testCase) => formatWorkflowCommand({
-      command: "error",
-      properties: {
-        file: testCase.displayFile,
-        line: String(testCase.location.line),
-        col: String(testCase.location.column),
-        title: "vitest-evals"
-      },
-      message: formatWorkflowMessage(testCase)
-    })
-  );
-}
-function buildCheckAnnotations(report, options = {}) {
-  const maxAnnotations = options.maxAnnotations ?? DEFAULT_MAX_CHECK_ANNOTATIONS;
-  return report.failures.filter(hasAnnotationLocation).slice(0, maxAnnotations).map((testCase) => ({
-    path: testCase.displayFile,
-    start_line: testCase.location.line,
-    end_line: testCase.location.line,
-    annotation_level: "failure",
-    title: truncate(
-      `${testCase.primaryFailure?.judgeName ?? "vitest-evals"} - ${testCase.displayName}`,
-      255
-    ),
-    message: truncate(
-      formatWorkflowMessage(testCase),
-      MAX_CHECK_FIELD_LENGTH
-    ),
-    raw_details: truncate(formatRawDetails(testCase), MAX_CHECK_FIELD_LENGTH)
-  }));
-}
-function hasAnnotationLocation(testCase) {
-  return Boolean(testCase.location);
-}
-function formatWorkflowMessage(testCase) {
-  const failure = testCase.primaryFailure;
-  const parts = [
-    testCase.displayName,
-    `score ${formatScore(failure?.score ?? testCase.eval?.avgScore)}`
-  ];
-  if (failure?.judgeName) {
-    parts.push(failure.judgeName);
-  }
-  const reason = compactLine(failure?.reason ?? "", 320);
-  if (reason) {
-    parts.push(reason);
-  }
-  return parts.join(" - ");
-}
-function formatRawDetails(testCase) {
-  const lines = [
-    `Test: ${testCase.displayName}`,
-    `Location: ${testCase.displayFile}:${testCase.location.line}`,
-    `Harness: ${testCase.harness?.name ?? "n/a"}`,
-    `Score: ${formatScore(testCase.primaryFailure?.score ?? testCase.eval?.avgScore)}`,
-    `Judge: ${testCase.primaryFailure?.judgeName ?? "n/a"}`,
-    "",
-    "Reason:",
-    testCase.primaryFailure?.reason ?? "n/a"
-  ];
-  const finalOutput = testCase.eval?.output ?? testCase.harness?.output;
-  if (finalOutput !== void 0) {
-    lines.push("", "Final:", stringifyValue(finalOutput, 8e3));
-  }
-  if (testCase.harness?.toolCalls.length) {
-    lines.push("", "Tools:");
-    for (const toolCall of testCase.harness.toolCalls) {
-      lines.push(
-        `- ${toolCall.name}: ${toolCall.error ? `error: ${toolCall.error}` : "ok"}`
-      );
-    }
-  }
-  return lines.join("\n");
-}
-function formatWorkflowCommand({
-  command,
-  properties,
-  message
-}) {
-  const renderedProperties = Object.entries(properties).map(([key, value]) => `${key}=${escapeCommandProperty(value)}`).join(",");
-  return `::${command} ${renderedProperties}::${escapeCommandData(message)}`;
-}
-
 // src/summary.ts
 var DEFAULT_MAX_FAILURES = 20;
 var DEFAULT_MAX_REASON_CHARS = 8e3;
@@ -544,14 +549,6 @@ function formatCountLine(passed, failed, total) {
 function renderSummaryTable(report, nonEvalFailures) {
   const rows = [
     ["Status", report.status],
-    [
-      "Tests",
-      formatCountLine(
-        report.totals.passed,
-        report.totals.failed,
-        report.totals.total
-      )
-    ],
     [
       "Evals",
       formatCountLine(
@@ -564,10 +561,6 @@ function renderSummaryTable(report, nonEvalFailures) {
   if (report.score) {
     rows.push(["Score", formatScoreSummary(report.score)]);
   }
-  const usage2 = formatUsage(report.usage);
-  if (usage2) {
-    rows.push(["Usage", usage2]);
-  }
   if (nonEvalFailures > 0) {
     rows.push([
       "Other Failures",
@@ -606,7 +599,7 @@ function renderScoreDistribution(report) {
   }
   const maxCount = Math.max(...counts);
   return [
-    "Score distribution",
+    "## Scores",
     "",
     "```text",
     ...SCORE_DISTRIBUTION_BUCKETS.map(
@@ -761,21 +754,6 @@ function renderAsciiTable(headers, rows) {
     ...rows.map(renderRow)
   ];
 }
-function formatUsage(usage2) {
-  const parts = [];
-  if (usage2.totalTokens > 0) {
-    parts.push(`${formatNumber(usage2.totalTokens)} tokens`);
-  }
-  if (usage2.toolCalls > 0) {
-    parts.push(
-      `${formatNumber(usage2.toolCalls)} tool${usage2.toolCalls === 1 ? "" : "s"}`
-    );
-  }
-  if (usage2.estimatedCost > 0) {
-    parts.push(`$${usage2.estimatedCost.toFixed(4)}`);
-  }
-  return parts.join(", ");
-}
 function formatCaseUsage(testCase) {
   const usage2 = testCase.harness?.usage;
   const parts = [];
@@ -880,29 +858,211 @@ function truncateCheckSummary(summary) {
   return `${summary.slice(0, MAX_CHECK_SUMMARY_LENGTH - CHECK_SUMMARY_TRUNCATION_SUFFIX.length).trimEnd()}${CHECK_SUMMARY_TRUNCATION_SUFFIX}`;
 }
 
-// src/cli.ts
-main().catch((error) => {
-  console.error(error instanceof Error ? error.message : String(error));
-  process.exitCode = 1;
-});
-async function main() {
-  const options = parseCliArgs(process.argv.slice(2));
-  if (options.help) {
-    console.log(usage());
-    return;
+// src/merge.ts
+function mergeEvalReports(reports) {
+  const cases = reports.flatMap((report) => report.cases);
+  const failures = reports.flatMap((report) => report.failures);
+  const scoredCases = cases.map((testCase) => testCase.eval?.avgScore).filter(
+    (score) => typeof score === "number" && Number.isFinite(score)
+  );
+  const startedAtValues = reports.map((report) => report.startedAt).filter(
+    (startedAt2) => typeof startedAt2 === "number" && Number.isFinite(startedAt2)
+  );
+  const startedAt = startedAtValues.length > 0 ? Math.min(...startedAtValues) : void 0;
+  return {
+    status: reports.some((report) => report.status === "failed") ? "failed" : "passed",
+    startedAt,
+    durationMs: mergeDuration(reports),
+    totals: {
+      total: sum(reports, (report) => report.totals.total),
+      passed: sum(reports, (report) => report.totals.passed),
+      failed: sum(reports, (report) => report.totals.failed),
+      skipped: sum(reports, (report) => report.totals.skipped),
+      evalTotal: sum(reports, (report) => report.totals.evalTotal),
+      evalPassed: sum(reports, (report) => report.totals.evalPassed),
+      evalFailed: sum(reports, (report) => report.totals.evalFailed)
+    },
+    score: scoredCases.length > 0 ? {
+      average: scoredCases.reduce((total, score) => total + score, 0) / scoredCases.length,
+      minimum: Math.min(...scoredCases)
+    } : void 0,
+    usage: mergeUsage(reports.map((report) => report.usage)),
+    cases,
+    failures
+  };
+}
+function mergeUsage(usages) {
+  return {
+    inputTokens: sum(usages, (usage2) => usage2.inputTokens),
+    outputTokens: sum(usages, (usage2) => usage2.outputTokens),
+    reasoningTokens: sum(usages, (usage2) => usage2.reasoningTokens),
+    totalTokens: sum(usages, (usage2) => usage2.totalTokens),
+    toolCalls: sum(usages, (usage2) => usage2.toolCalls)
+  };
+}
+function mergeDuration(reports) {
+  const intervals = reports.map((report) => {
+    if (typeof report.startedAt !== "number" || !Number.isFinite(report.startedAt) || typeof report.durationMs !== "number" || !Number.isFinite(report.durationMs)) {
+      return void 0;
+    }
+    return {
+      start: report.startedAt,
+      end: report.startedAt + report.durationMs
+    };
+  }).filter(
+    (interval) => Boolean(interval)
+  );
+  if (intervals.length > 0) {
+    return Math.max(...intervals.map((interval) => interval.end)) - Math.min(...intervals.map((interval) => interval.start));
   }
-  const json = JSON.parse(
-    await (0, import_promises.readFile)(options.jsonPath, "utf8")
+  const durations = reports.map((report) => report.durationMs).filter(
+    (durationMs) => typeof durationMs === "number" && Number.isFinite(durationMs)
   );
-  const report = collectEvalReport(json, {
-    workspace: options.workspace ?? process.env.GITHUB_WORKSPACE ?? process.cwd()
+  return durations.length > 0 ? durations.reduce((total, durationMs) => total + durationMs, 0) : void 0;
+}
+function sum(items, select) {
+  return items.reduce((total, item) => total + select(item), 0);
+}
+
+// src/results.ts
+var import_promises = require("fs/promises");
+var import_node_path2 = require("path");
+var GLOB_META_PATTERN = /[*?]/;
+async function resolveResultFiles(patterns, options = {}) {
+  const cwd = options.cwd ?? process.cwd();
+  const files = [];
+  for (const pattern of patterns.map((entry) => entry.trim()).filter(Boolean)) {
+    if (hasGlob(pattern)) {
+      files.push(...await expandGlob(pattern, cwd));
+    } else {
+      files.push((0, import_node_path2.isAbsolute)(pattern) ? pattern : (0, import_node_path2.resolve)(cwd, pattern));
+    }
+  }
+  return [...new Set(files)].sort();
+}
+function hasGlob(pattern) {
+  return GLOB_META_PATTERN.test(pattern);
+}
+async function expandGlob(pattern, cwd) {
+  const normalizedPattern = normalizeGlobPattern(pattern);
+  const absolutePattern = (0, import_node_path2.isAbsolute)(pattern);
+  const base = globBase(normalizedPattern);
+  const basePath = absolutePattern ? base || import_node_path2.sep : (0, import_node_path2.resolve)(cwd, base || ".");
+  const regex = globToRegExp(normalizedPattern);
+  const matches = [];
+  for (const file of await listFiles(basePath)) {
+    const normalizedFile = normalizePath(file);
+    const candidate = absolutePattern ? normalizedFile : normalizePath((0, import_node_path2.relative)((0, import_node_path2.resolve)(cwd), file));
+    if (regex.test(candidate)) {
+      matches.push(file);
+    }
+  }
+  return matches;
+}
+function globBase(pattern) {
+  const segments = pattern.split("/");
+  const baseSegments = [];
+  for (const segment of segments) {
+    if (hasGlob(segment)) {
+      break;
+    }
+    baseSegments.push(segment);
+  }
+  return baseSegments.join("/");
+}
+async function listFiles(directory) {
+  const entries = await readDirectory(directory);
+  if (!entries) {
+    return [];
+  }
+  const files = [];
+  for (const entry of entries) {
+    const child = (0, import_node_path2.resolve)(directory, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...await listFiles(child));
+    } else if (entry.isFile()) {
+      files.push(child);
+    }
+  }
+  return files;
+}
+async function readDirectory(directory) {
+  try {
+    return await (0, import_promises.readdir)(directory, { withFileTypes: true });
+  } catch {
+    return void 0;
+  }
+}
+function globToRegExp(pattern) {
+  let source = "";
+  for (let index = 0; index < pattern.length; index += 1) {
+    const char = pattern[index];
+    const next = pattern[index + 1];
+    if (char === "*" && next === "*") {
+      const following = pattern[index + 2];
+      if (following === "/") {
+        source += "(?:.*/)?";
+        index += 2;
+      } else {
+        source += ".*";
+        index += 1;
+      }
+      continue;
+    }
+    if (char === "*") {
+      source += "[^/]*";
+      continue;
+    }
+    if (char === "?") {
+      source += "[^/]";
+      continue;
+    }
+    source += escapeRegExp(char ?? "");
+  }
+  return new RegExp(`^${source}$`);
+}
+function escapeRegExp(value) {
+  return value.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
+}
+function normalizePath(path) {
+  return path.replace(/\\/g, "/");
+}
+function normalizeGlobPattern(pattern) {
+  const normalizedPattern = normalizePath(pattern);
+  if ((0, import_node_path2.isAbsolute)(pattern)) {
+    return normalizedPattern;
+  }
+  return normalizedPattern.replace(/^(\.\/)+/, "");
+}
+
+// src/report.ts
+async function publishEvalReport(options) {
+  const resultFiles = await resolveResultFiles(options.resultPatterns, {
+    cwd: options.cwd
   });
+  if (resultFiles.length === 0) {
+    throw new Error(
+      `No eval result files matched: ${options.resultPatterns.join(", ")}`
+    );
+  }
+  const reports = await Promise.all(
+    resultFiles.map(async (resultFile) => {
+      const json = await readVitestJsonReport(resultFile);
+      return collectEvalReport(json, {
+        workspace: options.workspace
+      });
+    })
+  );
+  const report = mergeEvalReports(reports);
   const summary = renderJobSummary(report, {
-    maxFailures: options.maxFailures
+    maxFailures: options.maxFailures,
+    maxOutputChars: options.maxOutputChars,
+    maxReasonChars: options.maxReasonChars,
+    maxToolCalls: options.maxToolCalls
   });
-  if (options.summaryEnabled) {
+  if (options.summaryEnabled !== false) {
     if (options.summaryPath) {
-      await (0, import_promises.appendFile)(options.summaryPath, `${summary}
+      await (0, import_promises2.appendFile)(options.summaryPath, `${summary}
 `);
     } else {
       console.log(summary);
@@ -915,28 +1075,81 @@ async function main() {
       console.log(command);
     }
   }
+  let checkRun;
   if (options.checkRun) {
     try {
-      const result = await publishCheckRun(report, {
+      checkRun = await publishCheckRun(report, {
         checkRunId: options.checkRunId,
         maxAnnotations: options.maxAnnotations,
         maxFailures: options.maxFailures,
+        maxOutputChars: options.maxOutputChars,
+        maxReasonChars: options.maxReasonChars,
+        maxToolCalls: options.maxToolCalls,
         name: options.checkName,
         repository: options.repository,
         sha: options.sha,
         token: options.token
       });
-      if (result.status === "skipped") {
-        warn(`GitHub Check Run skipped: ${result.reason}`);
+      if (checkRun.status === "skipped") {
+        options.warn?.(`GitHub Check Run skipped: ${checkRun.reason}`);
       }
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       if (options.failOnCheckError) {
         throw error;
       }
-      warn(message);
+      options.warn?.(message);
     }
   }
+  return {
+    report,
+    resultFiles,
+    checkRun
+  };
+}
+async function readVitestJsonReport(resultFile) {
+  try {
+    return JSON.parse(await (0, import_promises2.readFile)(resultFile, "utf8"));
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new Error(
+      `Failed to read eval result file ${resultFile}: ${message}`
+    );
+  }
+}
+
+// src/cli.ts
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exitCode = 1;
+});
+async function main() {
+  const options = parseCliArgs(process.argv.slice(2));
+  if (options.help) {
+    console.log(usage());
+    return;
+  }
+  const result = await publishEvalReport({
+    resultPatterns: options.resultPatterns,
+    cwd: options.workspace ?? process.env.GITHUB_WORKSPACE ?? process.cwd(),
+    workspace: options.workspace ?? process.env.GITHUB_WORKSPACE ?? process.cwd(),
+    summaryEnabled: options.summaryEnabled,
+    summaryPath: options.summaryPath,
+    annotations: options.annotations,
+    checkRun: options.checkRun,
+    checkRunId: options.checkRunId,
+    checkName: options.checkName,
+    failOnCheckError: options.failOnCheckError,
+    maxAnnotations: options.maxAnnotations,
+    maxFailures: options.maxFailures,
+    repository: options.repository,
+    sha: options.sha,
+    token: options.token,
+    warn
+  });
+  if (options.failOnFailures && result.report.status === "failed") {
+    process.exitCode = 1;
+  }
 }
 function warn(message) {
   if (process.env.GITHUB_ACTIONS === "true") {
@@ -947,15 +1160,16 @@ function warn(message) {
 }
 function usage() {
   return [
-    "Usage: vitest-evals-github-report [vitest-results.json] [--json <path>]",
+    "Usage: vitest-evals-github-report [vitest-results.json ...] [--json <path>]",
     "",
     "Options:",
-    "  --json <path>             Read Vitest JSON report from this path",
+    "  --json <path>             Read a Vitest JSON report from this path or glob",
     "  --summary <path>          Write job summary markdown to this path",
     "  --no-summary             Disable summary output",
     "  --annotations            Emit GitHub workflow-command annotations",
     "  --no-annotations         Disable workflow-command annotations",
     "  --check-run              Publish a GitHub Check Run when configured",
+    "  --fail-on-failures       Exit non-zero when the combined report failed",
     "  --fail-on-check-error    Fail when Check Run publishing fails",
     "  --check-run-id <id>      Update an existing Check Run",
     "  --check-name <name>      Check Run name (default: vitest-evals)",