@vitest-evals/github-reporter 0.15.0 → 0.16.1

package/dist/index.d.ts

@@ -80,9 +80,63 @@ type CollectOptions = {
 /** Converts a Vitest JSON report into the compact eval report model. */
 declare function collectEvalReport(input: VitestJsonReport, options?: CollectOptions): EvalReport;
 
+/** Optional floors and legacy fail-on-any-failure policy for a combined report. */
+type EvalGatePolicy = {
+    /**
+     * Minimum fraction of eval cases that must pass, in the range `[0, 1]`.
+     * Example: `0.8` requires at least 80% of eval cases to pass.
+     */
+    minPassRate?: number;
+    /**
+     * Minimum average eval score across scored cases, in the range `[0, 1]`.
+     */
+    minScoreAverage?: number;
+    /**
+     * When true, require every eval case to pass. Equivalent to setting
+     * `minPassRate` to `1`, and combined with an explicit floor by taking the
+     * stricter value.
+     */
+    failOnFailures?: boolean;
+};
+/** Result of applying a gate policy to a combined eval report. */
+type EvalGateResult = {
+    /** Whether the configured policy accepts this report. */
+    ok: boolean;
+    /** Compact status used by action outputs and summaries. */
+    status: "passed" | "failed";
+    /** True when any gate input was configured. */
+    enforced: boolean;
+    /** Eval pass rate when the report includes at least one eval case. */
+    passRate: number | null;
+    /** Short title for Check Runs and workflow annotations. */
+    title: string;
+    /** Longer explanation used in logs, summaries, and annotations. */
+    message: string;
+};
+/**
+ * Decide whether a combined eval report meets the configured CI gate.
+ *
+ * With no policy inputs, the result mirrors the report status and is not
+ * treated as an enforced gate. With floors or `failOnFailures`, non-eval
+ * failures and empty eval reports fail closed, and qualitative misses only
+ * fail when they break the configured floors.
+ */
+declare function evaluateEvalGate(report: EvalReport, policy?: EvalGatePolicy): EvalGateResult;
+/** Compute the eval pass rate for a report, or null when there are no evals. */
+declare function computePassRate(report: EvalReport): number | null;
+/** Format a 0-1 ratio as a percentage with one decimal place. */
+declare function formatPercent(value: number | null | undefined): string;
+/** Build a suite-level workflow annotation for a failed enforced gate. */
+declare function renderGateWorkflowCommand(gate: EvalGateResult): string | undefined;
+
 /** Options for limiting rendered GitHub annotations. */
 type AnnotationOptions = {
     maxAnnotations?: number;
+    /**
+     * Suite gate decision. When the gate passed, quality misses are demoted to
+     * warnings so a green score-gated check is not flooded with red failures.
+     */
+    gate?: EvalGateResult;
 };
 /** GitHub Check Run annotation payload. */
 type CheckAnnotation = {
@@ -105,6 +159,7 @@ type SummaryOptions = {
     maxReasonChars?: number;
     maxOutputChars?: number;
     maxToolCalls?: number;
+    gate?: EvalGateResult;
 };
 /** Renders the GitHub Actions job summary markdown for an eval report. */
 declare function renderJobSummary(report: EvalReport, options?: SummaryOptions): string;
@@ -116,8 +171,15 @@ type PublishCheckRunOptions = SummaryOptions & {
     sha?: string;
     name?: string;
     apiUrl?: string;
+    detailsUrl?: string;
+    externalId?: string;
     checkRunId?: number;
     maxAnnotations?: number;
+    /**
+     * Suite gate decision. When omitted, the report is evaluated with the
+     * default advisory policy so title/conclusion stay consistent.
+     */
+    gate?: EvalGateResult;
 };
 /** Result of attempting to publish a GitHub Check Run. */
 type PublishCheckRunResult = {
@@ -127,8 +189,70 @@ type PublishCheckRunResult = {
     status: "created" | "updated";
     id?: number;
     htmlUrl?: string;
+    sha?: string;
 };
+/**
+ * Resolve the commit SHA a Check Run should attach to.
+ *
+ * On `pull_request`, `GITHUB_SHA` is the temporary merge commit. PR status and
+ * required checks attach to the head commit, so prefer an explicit head SHA
+ * (option/env or `pull_request.head.sha` from the event payload) first.
+ */
+declare function resolveCheckSha(env?: NodeJS.ProcessEnv, options?: {
+    sha?: string;
+    eventPath?: string;
+}): string | undefined;
+/**
+ * Build a Check Run details URL that points back at the current workflow job
+ * (or run) when GitHub Actions env is present.
+ */
+declare function resolveCheckDetailsUrl(env?: NodeJS.ProcessEnv, options?: {
+    detailsUrl?: string;
+}): string | undefined;
 /** Publishes the eval report to a GitHub Check Run when configuration allows it. */
 declare function publishCheckRun(report: EvalReport, options?: PublishCheckRunOptions): Promise<PublishCheckRunResult>;
 
-export { type AnnotationOptions, type CheckAnnotation, type EvalCase, type EvalFailure, type EvalReport, type PublishCheckRunOptions, type PublishCheckRunResult, type SummaryOptions, buildCheckAnnotations, collectEvalReport, publishCheckRun, renderJobSummary, renderWorkflowCommands };
+/** Options for reading eval JSON files and publishing GitHub report surfaces. */
+type PublishEvalReportOptions = SummaryOptions & EvalGatePolicy & {
+    resultPatterns: string[];
+    cwd?: string;
+    workspace?: string;
+    summaryEnabled?: boolean;
+    summaryPath?: string;
+    annotations?: boolean;
+    checkRun?: boolean;
+    failOnCheckError?: boolean;
+    /**
+     * When true, keep the workflow step green on a failed gate so the Check
+     * Run owns PR status instead of a canned job failure line. Defaults to
+     * true only when a Check Run is actually published; if publishing is
+     * skipped/fails, the step still exits non-zero on a failed gate.
+     */
+    softFail?: boolean;
+    maxAnnotations?: number;
+    checkRunId?: number;
+    checkName?: string;
+    token?: string;
+    repository?: string;
+    sha?: string;
+    detailsUrl?: string;
+    externalId?: string;
+    warn?: (message: string) => void;
+};
+/** Result from publishing eval reports, including merged report data. */
+type PublishEvalReportResult = {
+    report: EvalReport;
+    resultFiles: string[];
+    gate: EvalGateResult;
+    /**
+     * True when an enforced gate rejects the report and soft-fail did not keep
+     * the step green. Advisory (ungated) runs never fail the step, matching
+     * `fail-on-failures: false` defaults.
+     */
+    shouldFail: boolean;
+    checkRun?: PublishCheckRunResult;
+};
+/** Reads, merges, and publishes eval reports to GitHub Actions surfaces. */
+declare function publishEvalReport(options: PublishEvalReportOptions): Promise<PublishEvalReportResult>;
+
+export { type AnnotationOptions, type CheckAnnotation, type EvalCase, type EvalFailure, type EvalGatePolicy, type EvalGateResult, type EvalReport, type PublishCheckRunOptions, type PublishCheckRunResult, type PublishEvalReportOptions, type PublishEvalReportResult, type SummaryOptions, buildCheckAnnotations, collectEvalReport, computePassRate, evaluateEvalGate, formatPercent, publishCheckRun, publishEvalReport, renderGateWorkflowCommand, renderJobSummary, renderWorkflowCommands, resolveCheckDetailsUrl, resolveCheckSha };