@vitest-evals/github-reporter 0.16.0 → 0.17.0

package/dist/index.d.ts

@@ -53,7 +53,10 @@ type EvalReport = {
         average: number;
         minimum?: number;
     };
-    usage: Required<UsageSummary>;
+    /** Application and other upstream usage. */
+    usage: AggregatedUsageSummary;
+    /** Judge-model usage. */
+    judgeUsage: AggregatedUsageSummary;
     cases: EvalCase[];
     failures: EvalCase[];
 };
@@ -63,8 +66,11 @@ type UsageSummary = {
     outputTokens?: number;
     reasoningTokens?: number;
     totalTokens?: number;
+    costUsd?: number;
     toolCalls?: number;
 };
+/** Aggregated usage; cost stays absent when no run reported one. */
+type AggregatedUsageSummary = Required<Omit<UsageSummary, "costUsd">> & Pick<UsageSummary, "costUsd">;
 /** Tool-call summary shown in reporter output. */
 type ToolCallSummary = {
     name: string;
@@ -171,6 +177,8 @@ type PublishCheckRunOptions = SummaryOptions & {
     sha?: string;
     name?: string;
     apiUrl?: string;
+    detailsUrl?: string;
+    externalId?: string;
     checkRunId?: number;
     maxAnnotations?: number;
     /**
@@ -187,7 +195,26 @@ type PublishCheckRunResult = {
     status: "created" | "updated";
     id?: number;
     htmlUrl?: string;
+    sha?: string;
 };
+/**
+ * Resolve the commit SHA a Check Run should attach to.
+ *
+ * On `pull_request`, `GITHUB_SHA` is the temporary merge commit. PR status and
+ * required checks attach to the head commit, so prefer an explicit head SHA
+ * (option/env or `pull_request.head.sha` from the event payload) first.
+ */
+declare function resolveCheckSha(env?: NodeJS.ProcessEnv, options?: {
+    sha?: string;
+    eventPath?: string;
+}): string | undefined;
+/**
+ * Build a Check Run details URL that points back at the current workflow job
+ * (or run) when GitHub Actions env is present.
+ */
+declare function resolveCheckDetailsUrl(env?: NodeJS.ProcessEnv, options?: {
+    detailsUrl?: string;
+}): string | undefined;
 /** Publishes the eval report to a GitHub Check Run when configuration allows it. */
 declare function publishCheckRun(report: EvalReport, options?: PublishCheckRunOptions): Promise<PublishCheckRunResult>;
 
@@ -201,12 +228,21 @@ type PublishEvalReportOptions = SummaryOptions & EvalGatePolicy & {
     annotations?: boolean;
     checkRun?: boolean;
     failOnCheckError?: boolean;
+    /**
+     * When true, keep the workflow step green on a failed gate so the Check
+     * Run owns PR status instead of a canned job failure line. Defaults to
+     * true only when a Check Run is actually published; if publishing is
+     * skipped/fails, the step still exits non-zero on a failed gate.
+     */
+    softFail?: boolean;
     maxAnnotations?: number;
     checkRunId?: number;
     checkName?: string;
     token?: string;
     repository?: string;
     sha?: string;
+    detailsUrl?: string;
+    externalId?: string;
     warn?: (message: string) => void;
 };
 /** Result from publishing eval reports, including merged report data. */
@@ -215,8 +251,9 @@ type PublishEvalReportResult = {
     resultFiles: string[];
     gate: EvalGateResult;
     /**
-     * True when an enforced gate rejects the report. Advisory (ungated) runs
-     * never fail the step, matching `fail-on-failures: false` defaults.
+     * True when an enforced gate rejects the report and soft-fail did not keep
+     * the step green. Advisory (ungated) runs never fail the step, matching
+     * `fail-on-failures: false` defaults.
      */
     shouldFail: boolean;
     checkRun?: PublishCheckRunResult;
@@ -224,4 +261,4 @@ type PublishEvalReportResult = {
 /** Reads, merges, and publishes eval reports to GitHub Actions surfaces. */
 declare function publishEvalReport(options: PublishEvalReportOptions): Promise<PublishEvalReportResult>;
 
-export { type AnnotationOptions, type CheckAnnotation, type EvalCase, type EvalFailure, type EvalGatePolicy, type EvalGateResult, type EvalReport, type PublishCheckRunOptions, type PublishCheckRunResult, type PublishEvalReportOptions, type PublishEvalReportResult, type SummaryOptions, buildCheckAnnotations, collectEvalReport, computePassRate, evaluateEvalGate, formatPercent, publishCheckRun, publishEvalReport, renderGateWorkflowCommand, renderJobSummary, renderWorkflowCommands };
+export { type AnnotationOptions, type CheckAnnotation, type EvalCase, type EvalFailure, type EvalGatePolicy, type EvalGateResult, type EvalReport, type PublishCheckRunOptions, type PublishCheckRunResult, type PublishEvalReportOptions, type PublishEvalReportResult, type SummaryOptions, buildCheckAnnotations, collectEvalReport, computePassRate, evaluateEvalGate, formatPercent, publishCheckRun, publishEvalReport, renderGateWorkflowCommand, renderJobSummary, renderWorkflowCommands, resolveCheckDetailsUrl, resolveCheckSha };

package/dist/index.js

@@ -29,7 +29,9 @@ __export(index_exports, {
   publishEvalReport: () => publishEvalReport,
   renderGateWorkflowCommand: () => renderGateWorkflowCommand,
   renderJobSummary: () => renderJobSummary,
-  renderWorkflowCommands: () => renderWorkflowCommands
+  renderWorkflowCommands: () => renderWorkflowCommands,
+  resolveCheckDetailsUrl: () => resolveCheckDetailsUrl,
+  resolveCheckSha: () => resolveCheckSha
 });
 module.exports = __toCommonJS(index_exports);
 
@@ -126,7 +128,8 @@ function collectEvalReport(input, options = {}) {
   const cases = workspace.cases.map(collectEvalCase);
   const failures = cases.filter((testCase) => testCase.status === "failed");
   const evalScores = cases.map((testCase) => testCase.eval?.avgScore).filter((score) => isFiniteNumber(score));
-  const usage = sumUsage(cases);
+  const usage = sumAppUsage(cases);
+  const judgeUsage = sumJudgeUsage(cases);
   const durationMs = workspace.runs[0]?.durationMs;
   return {
     status: input.success && failures.length === 0 ? "passed" : "failed",
@@ -146,6 +149,7 @@ function collectEvalReport(input, options = {}) {
       minimum: Math.min(...evalScores)
     } : void 0,
     usage,
+    judgeUsage,
     cases,
     failures
   };
@@ -270,24 +274,67 @@ function stringifyReason(value) {
   }
   return typeof value === "string" ? value : stringifyValue(value, 4e3);
 }
-function sumUsage(cases) {
-  const usage = {
+function emptyUsage() {
+  return {
     inputTokens: 0,
     outputTokens: 0,
     reasoningTokens: 0,
     totalTokens: 0,
     toolCalls: 0
   };
+}
+function addRunUsage(total, usage) {
+  total.inputTokens += usage?.inputTokens ?? 0;
+  total.outputTokens += usage?.outputTokens ?? 0;
+  total.reasoningTokens += usage?.reasoningTokens ?? 0;
+  total.totalTokens += usage?.totalTokens ?? (usage?.inputTokens ?? 0) + (usage?.outputTokens ?? 0) + (usage?.reasoningTokens ?? 0);
+  if (usage?.costUsd !== void 0) {
+    total.costUsd = (total.costUsd ?? 0) + usage.costUsd;
+  }
+  total.toolCalls += usage?.toolCalls ?? 0;
+}
+function sumAppUsage(cases) {
+  const usage = emptyUsage();
+  const runUsages = cases.map(appUsageForCase).filter((item) => item !== void 0);
+  for (const runUsage of runUsages) {
+    addRunUsage(usage, runUsage);
+  }
+  omitPartialCost(usage, runUsages);
+  return usage;
+}
+function appUsageForCase(testCase) {
+  const usage = testCase.harness?.usage;
+  const effectiveToolCalls = toolCallCount(testCase);
+  if (!usage && effectiveToolCalls === 0) {
+    return void 0;
+  }
+  return {
+    ...usage,
+    ...effectiveToolCalls > 0 ? { toolCalls: effectiveToolCalls } : {}
+  };
+}
+function sumJudgeUsage(cases) {
+  const usage = emptyUsage();
+  const runUsages = [];
   for (const testCase of cases) {
-    const caseUsage = testCase.harness?.usage;
-    usage.inputTokens += caseUsage?.inputTokens ?? 0;
-    usage.outputTokens += caseUsage?.outputTokens ?? 0;
-    usage.reasoningTokens += caseUsage?.reasoningTokens ?? 0;
-    usage.totalTokens += caseUsage?.totalTokens ?? (caseUsage?.inputTokens ?? 0) + (caseUsage?.outputTokens ?? 0) + (caseUsage?.reasoningTokens ?? 0);
-    usage.toolCalls += toolCallCount(testCase);
+    for (const score of testCase.eval?.scores ?? []) {
+      for (const run of score.judgeRuns ?? []) {
+        runUsages.push(run.usage);
+        addRunUsage(usage, run.usage);
+      }
+    }
   }
+  omitPartialCost(usage, runUsages);
   return usage;
 }
+function omitPartialCost(total, usages) {
+  if (usages.some(hasUsageWithoutCost)) {
+    total.costUsd = void 0;
+  }
+}
+function hasUsageWithoutCost(usage) {
+  return usage?.costUsd === void 0 && ((usage?.totalTokens ?? (usage?.inputTokens ?? 0) + (usage?.outputTokens ?? 0) + (usage?.reasoningTokens ?? 0)) > 0 || (usage?.toolCalls ?? 0) > 0);
+}
 function toolCallCount(testCase) {
   const usageToolCalls = testCase.harness?.usage?.toolCalls;
   if (usageToolCalls !== void 0) {
@@ -431,8 +478,8 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate,
-      title: "Eval report hard failure",
-      message: `${formatNumber(nonEvalFailures)} non-eval test failure${nonEvalFailures === 1 ? "" : "s"}; ${counts}`
+      title: "Eval run failed",
+      message: `${formatNumber(nonEvalFailures)} test failure${nonEvalFailures === 1 ? "" : "s"} outside eval cases; ${counts}`
     };
   }
   if (report.totals.evalTotal === 0) {
@@ -441,7 +488,7 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate: null,
-      title: "Eval report hard failure",
+      title: "Eval run failed",
       message: "no eval cases were reported"
     };
   }
@@ -451,8 +498,8 @@ function evaluateEvalGate(report, policy = {}) {
       status: "failed",
       enforced: true,
       passRate,
-      title: "Eval report hard failure",
-      message: `vitest run failed without counted test failures; ${counts}`
+      title: "Eval run failed",
+      message: `Vitest failed without reporting a failed test; ${counts}`
     };
   }
   if (minPassRate !== void 0 && (passRate === null || passRate + Number.EPSILON < minPassRate)) {
@@ -462,7 +509,7 @@ function evaluateEvalGate(report, policy = {}) {
       enforced: true,
       passRate,
       title: `Eval pass rate ${formatPercent(passRate)} \u2014 required ${formatPercent(minPassRate)}`,
-      message: `eval pass rate below floor: ${counts}; required >= ${formatPercent(minPassRate)}`
+      message: `pass rate is below the minimum: ${counts}; minimum ${formatPercent(minPassRate)}`
     };
   }
   if (minScoreAverage !== void 0) {
@@ -473,8 +520,8 @@ function evaluateEvalGate(report, policy = {}) {
         status: "failed",
         enforced: true,
         passRate,
-        title: "Eval score gate failed",
-        message: `no score average available; required avg score >= ${formatScore(minScoreAverage)}`
+        title: "Average score unavailable",
+        message: `no average score was reported; minimum ${formatScore(minScoreAverage)}`
       };
     }
     if (average + Number.EPSILON < minScoreAverage) {
@@ -483,8 +530,8 @@ function evaluateEvalGate(report, policy = {}) {
         status: "failed",
         enforced: true,
         passRate,
-        title: `Avg score ${formatScore(average)} \u2014 required ${formatScore(minScoreAverage)}`,
-        message: `avg score below floor: ${counts}; required avg score >= ${formatScore(minScoreAverage)}`
+        title: `Average score ${formatScore(average)} \u2014 required ${formatScore(minScoreAverage)}`,
+        message: `average score is below the minimum: ${counts}; minimum ${formatScore(minScoreAverage)}`
       };
     }
   }
@@ -494,7 +541,7 @@ function evaluateEvalGate(report, policy = {}) {
     enforced: true,
     passRate,
     title: enforcedPassTitle(report, passRate, minPassRate, minScoreAverage),
-    message: `eval gate passed: ${counts}${formatFloorSuffix(minPassRate, minScoreAverage)}`
+    message: `requirements met: ${counts}${formatMinimumSuffix(minPassRate, minScoreAverage)}`
   };
 }
 function computePassRate(report) {
@@ -527,7 +574,7 @@ function resolveMinPassRate(policy) {
 }
 function defaultCheckTitle(report) {
   if (report.failures.length === 0 && report.status === "passed") {
-    return "No eval failures";
+    return "No eval cases failed";
   }
   if (report.failures.length === 0) {
     return "Vitest run failed";
@@ -536,10 +583,10 @@ function defaultCheckTitle(report) {
 }
 function enforcedPassTitle(report, passRate, minPassRate, minScoreAverage) {
   if (minPassRate !== void 0) {
-    return `Eval pass rate ${formatPercent(passRate)} \u2014 floor ${formatPercent(minPassRate)}`;
+    return `Eval pass rate ${formatPercent(passRate)} \u2014 minimum ${formatPercent(minPassRate)}`;
   }
   if (minScoreAverage !== void 0) {
-    return `Avg score ${formatScore(report.score?.average)} \u2014 floor ${formatScore(minScoreAverage)}`;
+    return `Average score ${formatScore(report.score?.average)} \u2014 minimum ${formatScore(minScoreAverage)}`;
   }
   return defaultCheckTitle(report);
 }
@@ -548,19 +595,22 @@ function formatEvalCounts(report, passRate) {
   const passRateText = passRate === null ? "n/a" : formatPercent(passRate);
   return `${formatNumber(report.totals.evalPassed)}/${formatNumber(
     report.totals.evalTotal
-  )} passed (${passRateText}), avg score ${scoreText}`;
+  )} passed (${passRateText}), average score ${scoreText}`;
 }
-function formatFloorSuffix(minPassRate, minScoreAverage) {
+function formatMinimumSuffix(minPassRate, minScoreAverage) {
   const parts = [];
   if (minPassRate !== void 0) {
-    parts.push(`pass rate floor ${formatPercent(minPassRate)}`);
+    parts.push(`minimum pass rate ${formatPercent(minPassRate)}`);
   }
   if (minScoreAverage !== void 0) {
-    parts.push(`avg score floor ${formatScore(minScoreAverage)}`);
+    parts.push(`minimum average score ${formatScore(minScoreAverage)}`);
   }
   return parts.length === 0 ? "" : `; ${parts.join(", ")}`;
 }
 
+// src/github.ts
+var import_node_fs = require("fs");
+
 // src/summary.ts
 var DEFAULT_MAX_FAILURES = 20;
 var DEFAULT_MAX_REASON_CHARS = 8e3;
@@ -588,23 +638,23 @@ function renderJobSummary(report, options = {}) {
     ""
   ];
   if (report.failures.length > 0) {
-    const failureHeading = options.gate?.ok === true ? "### Quality Misses" : "### Failures";
+    const failureHeading = options.gate?.ok === true ? "### Cases Below Target" : "### Failures";
     lines.push(failureHeading, "");
     failures.forEach((testCase, index) => {
       lines.push(...renderFailureDetails(testCase, index + 1, options), "");
     });
     if (report.failures.length > failures.length) {
-      const omittedLabel = options.gate?.ok === true ? "quality misses" : "failures";
+      const omittedLabel = options.gate?.ok === true ? "cases below target" : "failures";
       lines.push(
         `${report.failures.length - failures.length} more ${omittedLabel} omitted from this summary.`,
         ""
       );
     }
   } else if (report.totals.evalTotal > 0) {
-    lines.push("### Failures", "", "No eval failures.", "");
+    lines.push("### Failures", "", "No eval cases failed.", "");
   }
   if (report.totals.evalTotal === 0) {
-    lines.push("No eval metadata was found in the Vitest JSON report.", "");
+    lines.push("No eval results were found in the Vitest JSON report.", "");
   }
   return `${lines.join("\n")}
 `;
@@ -614,9 +664,9 @@ function formatCountLine(passed, failed, total) {
 }
 function renderSummaryTable(report, nonEvalFailures, gate) {
   const rows = [
-    ["Status", gate?.status ?? report.status],
+    ["Status", capitalize(gate?.status ?? report.status)],
     [
-      "Evals",
+      "Eval cases",
       formatCountLine(
         report.totals.evalPassed,
         report.totals.evalFailed,
@@ -625,23 +675,31 @@ function renderSummaryTable(report, nonEvalFailures, gate) {
     ]
   ];
   if (gate?.passRate !== void 0 && gate.passRate !== null) {
-    rows.push(["Pass Rate", formatPercent(gate.passRate)]);
+    rows.push(["Pass rate", formatPercent(gate.passRate)]);
   } else if (report.totals.evalTotal > 0) {
     rows.push([
-      "Pass Rate",
+      "Pass rate",
       formatPercent(report.totals.evalPassed / report.totals.evalTotal)
     ]);
   }
   if (report.score) {
     rows.push(["Score", formatScoreSummary(report.score)]);
   }
+  if (hasUsage(report.usage) || hasUsage(report.judgeUsage)) {
+    rows.push(["App usage", formatUsage(report.usage)]);
+    rows.push(["Judge usage", formatUsage(report.judgeUsage)]);
+    rows.push([
+      "Total usage",
+      formatUsage(sumUsage(report.usage, report.judgeUsage))
+    ]);
+  }
   if (gate?.enforced) {
-    rows.push(["Gate", gate.message]);
+    rows.push(["Requirements", gate.message]);
   }
   if (nonEvalFailures > 0) {
     rows.push([
-      "Other Failures",
-      `${formatNumber(nonEvalFailures)} non-eval test failure${nonEvalFailures === 1 ? "" : "s"}`
+      "Other test failures",
+      `${formatNumber(nonEvalFailures)} test failure${nonEvalFailures === 1 ? "" : "s"} outside eval cases`
     ]);
   }
   rows.push(["Duration", formatDuration(report.durationMs)]);
@@ -654,7 +712,53 @@ function renderSummaryTable(report, nonEvalFailures, gate) {
   ];
 }
 function formatScoreSummary(score) {
-  return `avg ${formatScore(score.average)}${score.minimum === void 0 ? "" : `, min ${formatScore(score.minimum)}`}`;
+  return `average ${formatScore(score.average)}${score.minimum === void 0 ? "" : `, lowest ${formatScore(score.minimum)}`}`;
+}
+function capitalize(value) {
+  return value.charAt(0).toUpperCase() + value.slice(1);
+}
+function hasUsage(usage) {
+  return usage.totalTokens > 0 || usage.costUsd !== void 0 || usage.toolCalls > 0;
+}
+function sumUsage(app, judge) {
+  const appCostKnown = !hasUsageWithoutCost2(app);
+  const judgeCostKnown = !hasUsageWithoutCost2(judge);
+  return {
+    inputTokens: app.inputTokens + judge.inputTokens,
+    outputTokens: app.outputTokens + judge.outputTokens,
+    reasoningTokens: app.reasoningTokens + judge.reasoningTokens,
+    totalTokens: app.totalTokens + judge.totalTokens,
+    ...appCostKnown && judgeCostKnown ? { costUsd: (app.costUsd ?? 0) + (judge.costUsd ?? 0) } : {},
+    toolCalls: app.toolCalls + judge.toolCalls
+  };
+}
+function hasUsageWithoutCost2(usage) {
+  return usage.costUsd === void 0 && (usage.totalTokens > 0 || usage.toolCalls > 0);
+}
+function formatUsage(usage) {
+  const parts = [];
+  if (usage.totalTokens > 0) {
+    parts.push(`${formatNumber(usage.totalTokens)} tokens`);
+  }
+  if (usage.costUsd !== void 0) {
+    parts.push(
+      usage.costUsd.toLocaleString("en-US", {
+        style: "currency",
+        currency: "USD",
+        minimumFractionDigits: 2,
+        maximumFractionDigits: 6
+      })
+    );
+  }
+  if (hasUsageWithoutCost2(usage)) {
+    parts.push("cost unavailable");
+  }
+  if (usage.toolCalls > 0) {
+    parts.push(
+      `${formatNumber(usage.toolCalls)} tool call${usage.toolCalls === 1 ? "" : "s"}`
+    );
+  }
+  return parts.join(", ") || "none";
 }
 function escapeTableCell(value) {
   return value.replace(/\r?\n/g, " ").replace(/\\/g, "\\\\").replace(/\|/g, "\\|");
@@ -734,7 +838,7 @@ function renderFailureBlock(testCase, {
     ["Case", `${number}. ${testCase.displayName}`],
     ["Status", testCase.status],
     ["Location", formatLocation(testCase.displayFile, testCase.location)],
-    ["Harness", testCase.harness?.name ?? "n/a"],
+    ["App runner", testCase.harness?.name ?? "n/a"],
     ["Score", formatScore(failure?.score ?? testCase.eval?.avgScore)],
     ["Judge", failure?.judgeName ?? "n/a"]
   ];
@@ -772,7 +876,7 @@ function renderFailureBlock(testCase, {
   if (finalOutput !== void 0) {
     lines.push(
       ...renderAsciiSection(
-        "Final Output",
+        "Output",
         stringifyValue(finalOutput, maxOutputChars).split(/\r?\n/)
       ),
       ""
@@ -800,7 +904,7 @@ function renderFailureBlock(testCase, {
   if (testCase.harness?.errors.length) {
     lines.push(
       ...renderAsciiSection(
-        "Harness Errors",
+        "App errors",
         stringifyValue(testCase.harness.errors, maxReasonChars).split(/\r?\n/)
       ),
       ""
@@ -849,7 +953,9 @@ function formatCaseUsage(testCase) {
     parts.push(`${formatNumber(totalTokens)} tokens`);
   }
   if (toolCalls > 0) {
-    parts.push(`${formatNumber(toolCalls)} tool${toolCalls === 1 ? "" : "s"}`);
+    parts.push(
+      `${formatNumber(toolCalls)} tool call${toolCalls === 1 ? "" : "s"}`
+    );
   }
   if (testCase.harness?.timingMs !== void 0) {
     parts.push(formatDuration(testCase.harness.timingMs));
@@ -861,10 +967,44 @@ function formatCaseUsage(testCase) {
 var DEFAULT_CHECK_NAME = "vitest-evals";
 var MAX_CHECK_SUMMARY_LENGTH = 64e3;
 var CHECK_SUMMARY_TRUNCATION_SUFFIX = "\n\n[truncated for GitHub Check Run]\n";
+function resolveCheckSha(env = process.env, options = {}) {
+  const explicit = options.sha?.trim() || env.GITHUB_PR_HEAD_SHA?.trim();
+  if (explicit) {
+    return explicit;
+  }
+  const eventPath = options.eventPath?.trim() || env.GITHUB_EVENT_PATH?.trim();
+  if (eventPath) {
+    try {
+      const event = JSON.parse((0, import_node_fs.readFileSync)(eventPath, "utf8"));
+      const headSha = event.pull_request?.head?.sha;
+      if (typeof headSha === "string" && headSha.trim()) {
+        return headSha.trim();
+      }
+    } catch {
+    }
+  }
+  return env.GITHUB_SHA?.trim() || void 0;
+}
+function resolveCheckDetailsUrl(env = process.env, options = {}) {
+  const explicit = options.detailsUrl?.trim();
+  if (explicit) {
+    return explicit;
+  }
+  const server = env.GITHUB_SERVER_URL?.replace(/\/$/, "");
+  const repository = env.GITHUB_REPOSITORY?.trim();
+  const runId = env.GITHUB_RUN_ID?.trim();
+  if (!server || !repository || !runId) {
+    return void 0;
+  }
+  return `${server}/${repository}/actions/runs/${runId}`;
+}
 async function publishCheckRun(report, options = {}) {
   const token = options.token ?? process.env.GITHUB_TOKEN;
   const repository = options.repository ?? process.env.GITHUB_REPOSITORY;
-  const sha = options.sha ?? process.env.GITHUB_SHA;
+  const sha = resolveCheckSha(process.env, { sha: options.sha });
+  const detailsUrl = resolveCheckDetailsUrl(process.env, {
+    detailsUrl: options.detailsUrl
+  });
   if (!token) {
     return { status: "skipped", reason: "missing GITHUB_TOKEN" };
   }
@@ -872,7 +1012,10 @@ async function publishCheckRun(report, options = {}) {
     return { status: "skipped", reason: "missing GITHUB_REPOSITORY" };
   }
   if (!sha && options.checkRunId === void 0) {
-    return { status: "skipped", reason: "missing GITHUB_SHA" };
+    return {
+      status: "skipped",
+      reason: "missing commit SHA (set --sha / options.sha, GITHUB_PR_HEAD_SHA, pull_request.head.sha, or GITHUB_SHA)"
+    };
   }
   const [owner, repo] = repository.split("/");
   if (!owner || !repo) {
@@ -881,7 +1024,7 @@ async function publishCheckRun(report, options = {}) {
       reason: `invalid GitHub repository: ${repository}`
     };
   }
-  const payload = buildCheckRunPayload(report, options);
+  const payload = buildCheckRunPayload(report, options, detailsUrl);
   const apiUrl = options.apiUrl ?? process.env.GITHUB_API_URL ?? "https://api.github.com";
   const requestUrl = options.checkRunId === void 0 ? `${apiUrl}/repos/${owner}/${repo}/check-runs` : `${apiUrl}/repos/${owner}/${repo}/check-runs/${options.checkRunId}`;
   const response = await fetch(requestUrl, {
@@ -896,6 +1039,7 @@ async function publishCheckRun(report, options = {}) {
       options.checkRunId === void 0 ? {
         name: options.name ?? DEFAULT_CHECK_NAME,
         head_sha: sha,
+        ...options.externalId ? { external_id: options.externalId } : {},
         ...payload
       } : payload
     )
@@ -910,10 +1054,11 @@ async function publishCheckRun(report, options = {}) {
   return {
     status: options.checkRunId === void 0 ? "created" : "updated",
     id: data.id,
-    htmlUrl: data.html_url
+    htmlUrl: data.html_url,
+    sha
   };
 }
-function buildCheckRunPayload(report, options) {
+function buildCheckRunPayload(report, options, detailsUrl) {
   const gate = options.gate ?? evaluateEvalGate(report);
   const annotations = buildCheckAnnotations(report, {
     maxAnnotations: options.maxAnnotations,
@@ -923,6 +1068,7 @@ function buildCheckRunPayload(report, options) {
     status: "completed",
     conclusion: gate.ok ? "success" : "failure",
     completed_at: (/* @__PURE__ */ new Date()).toISOString(),
+    ...detailsUrl ? { details_url: detailsUrl } : {},
     output: {
       title: gate.title,
       summary: truncateCheckSummary(
@@ -979,16 +1125,22 @@ function mergeEvalReports(reports) {
       minimum: Math.min(...scoredCases)
     } : void 0,
     usage: mergeUsage(reports.map((report) => report.usage)),
+    judgeUsage: mergeUsage(reports.map((report) => report.judgeUsage)),
     cases,
     failures
   };
 }
 function mergeUsage(usages) {
+  const costs = usages.map((usage) => usage.costUsd).filter((cost) => cost !== void 0);
+  const costComplete = !usages.some(
+    (usage) => usage.costUsd === void 0 && (usage.totalTokens > 0 || usage.toolCalls > 0)
+  );
   return {
     inputTokens: sum(usages, (usage) => usage.inputTokens),
     outputTokens: sum(usages, (usage) => usage.outputTokens),
     reasoningTokens: sum(usages, (usage) => usage.reasoningTokens),
     totalTokens: sum(usages, (usage) => usage.totalTokens),
+    ...costComplete && costs.length > 0 ? { costUsd: costs.reduce((total, cost) => total + cost, 0) } : {},
     toolCalls: sum(usages, (usage) => usage.toolCalls)
   };
 }
@@ -1080,11 +1232,17 @@ async function publishEvalReport(options) {
         name: options.checkName,
         repository: options.repository,
         sha: options.sha,
+        detailsUrl: options.detailsUrl,
+        externalId: options.externalId,
         token: options.token,
         gate
       });
       if (checkRun.status === "skipped") {
         options.warn?.(`GitHub Check Run skipped: ${checkRun.reason}`);
+      } else if (checkRun.htmlUrl) {
+        console.log(`published check run: ${checkRun.htmlUrl}`);
+      } else if (checkRun.id !== void 0) {
+        console.log(`published check run id: ${checkRun.id}`);
       }
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -1094,14 +1252,21 @@ async function publishEvalReport(options) {
       options.warn?.(message);
     }
   }
+  const gateFailed = gate.enforced && !gate.ok;
+  const wantsSoftFail = options.softFail ?? options.checkRun === true;
+  const softFail = wantsSoftFail && checkRunPublished(checkRun);
+  const shouldFail = gateFailed && !softFail;
   return {
     report,
     resultFiles,
     gate,
-    shouldFail: gate.enforced && !gate.ok,
+    shouldFail,
     checkRun
   };
 }
+function checkRunPublished(checkRun) {
+  return checkRun?.status === "created" || checkRun?.status === "updated";
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   buildCheckAnnotations,
@@ -1113,6 +1278,8 @@ async function publishEvalReport(options) {
   publishEvalReport,
   renderGateWorkflowCommand,
   renderJobSummary,
-  renderWorkflowCommands
+  renderWorkflowCommands,
+  resolveCheckDetailsUrl,
+  resolveCheckSha
 });
 //# sourceMappingURL=index.js.map