@evalgate/sdk 2.0.0

package/dist/cli/formatters/github.js

@@ -0,0 +1,135 @@
+"use strict";
+/**
+ * GitHub formatter for evalgate check.
+ * - stdout: minimal (verdict + score + link) + ::error annotations for failed cases
+ * - Step summary: full Markdown written to GITHUB_STEP_SUMMARY (not stdout)
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.appendStepSummary = appendStepSummary;
+exports.formatGitHub = formatGitHub;
+const fs = __importStar(require("node:fs"));
+const snippet_1 = require("../render/snippet");
+const ANNOTATION_MAX = 10;
+function escapeAnnotationMessage(s) {
+    return s.replace(/\r/g, "").replace(/\n/g, "%0A");
+}
+function formatAnnotation(fc) {
+    const id = fc.testCaseId ?? fc.name ?? "unknown";
+    const reason = fc.reason ?? fc.outputSnippet ?? fc.output ?? "no output";
+    const msg = escapeAnnotationMessage(`TestCase ${id} failed - ${(0, snippet_1.truncateSnippet)(reason, 100)}`);
+    return `::error title=EvalGate regression::${msg}`;
+}
+function appendStepSummary(report) {
+    const path = typeof process !== "undefined" && process.env?.GITHUB_STEP_SUMMARY;
+    if (!path)
+        return;
+    const lines = [];
+    const passed = report.verdict === "pass";
+    const warned = report.verdict === "warn";
+    lines.push("## EvalGate Gate");
+    lines.push("");
+    lines.push(passed && !warned
+        ? "✅ **PASSED**"
+        : warned
+            ? `⚠️ **WARNED**: ${report.reasonMessage ?? report.reasonCode}`
+            : `❌ **FAILED**: ${report.reasonMessage ?? report.reasonCode}`);
+    lines.push("");
+    const deltaStr = report.baselineScore != null && report.delta != null
+        ? ` (baseline ${report.baselineScore}, ${report.delta >= 0 ? "+" : ""}${report.delta} pts)`
+        : "";
+    lines.push(`**Score:** ${report.score ?? 0}/100${deltaStr}`);
+    lines.push("");
+    const failedCases = report.failedCases ?? [];
+    if (failedCases.length > 0) {
+        lines.push(`### ${failedCases.length} failing case${failedCases.length === 1 ? "" : "s"}`);
+        lines.push("");
+        for (const fc of failedCases.slice(0, 10)) {
+            const label = fc.name ?? fc.input ?? "(unnamed)";
+            const exp = (0, snippet_1.truncateSnippet)(fc.expectedOutput ?? fc.expectedSnippet, 80);
+            const out = (0, snippet_1.truncateSnippet)(fc.output ?? fc.outputSnippet, 80);
+            const reason = out ? `got "${out}"` : "no output";
+            lines.push(`- **${(0, snippet_1.truncateSnippet)(label, 60)}** — expected: ${exp || "(unknown)"}, ${reason}`);
+        }
+        if (failedCases.length > 10) {
+            lines.push(`- _+ ${failedCases.length - 10} more_`);
+        }
+        lines.push("");
+    }
+    if (report.dashboardUrl) {
+        lines.push(`[View Dashboard](${report.dashboardUrl})`);
+        lines.push("");
+    }
+    if (!passed) {
+        lines.push("> **Tip:** Run `evalgate explain` locally to see root causes and suggested fixes.");
+        lines.push("> Report saved to `.evalgate/last-report.json` — upload as a build artifact for offline analysis.");
+        lines.push("");
+    }
+    try {
+        fs.appendFileSync(path, lines.join("\n"), "utf8");
+    }
+    catch {
+        // Non-fatal: step summary is best-effort
+    }
+}
+function formatGitHub(report) {
+    const stdoutLines = [];
+    // Emit ::error annotations for failed cases (up to N)
+    const failedCases = report.failedCases ?? [];
+    const toAnnotate = failedCases.slice(0, ANNOTATION_MAX);
+    for (const fc of toAnnotate) {
+        stdoutLines.push(formatAnnotation(fc));
+    }
+    // Minimal summary: verdict + score + link
+    const passed = report.verdict === "pass";
+    const warned = report.verdict === "warn";
+    const failReason = report.reasonMessage ?? report.reasonCode;
+    if (passed && !warned)
+        stdoutLines.push("\n✓ EvalGate gate PASSED");
+    else if (warned)
+        stdoutLines.push(`\n⚠ EvalGate gate WARNED: ${failReason}`);
+    else
+        stdoutLines.push(`\n✗ EvalGate gate FAILED: ${failReason}`);
+    const deltaStr = report.baselineScore != null && report.delta != null
+        ? ` (baseline ${report.baselineScore}, ${report.delta >= 0 ? "+" : ""}${report.delta} pts)`
+        : "";
+    stdoutLines.push(`Score: ${report.score ?? 0}/100${deltaStr}`);
+    if (report.dashboardUrl) {
+        stdoutLines.push(`Dashboard: ${report.dashboardUrl}`);
+    }
+    // Write full markdown to GITHUB_STEP_SUMMARY (not stdout)
+    appendStepSummary(report);
+    return stdoutLines.join("\n");
+}

package/dist/cli/formatters/human.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Human-readable formatter for evalgate check output.
+ * Deterministic: verdict → score → failures → link → hint.
+ */
+import type { CheckReport } from "./types";
+export declare function formatHuman(report: CheckReport): string;

package/dist/cli/formatters/human.js

@@ -0,0 +1,110 @@
+"use strict";
+/**
+ * Human-readable formatter for evalgate check output.
+ * Deterministic: verdict → score → failures → link → hint.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatHuman = formatHuman;
+const snippet_1 = require("../render/snippet");
+const TOP_N = 3;
+function formatHuman(report) {
+    const lines = [];
+    const passed = report.verdict === "pass";
+    const warned = report.verdict === "warn";
+    const failReason = report.reasonMessage;
+    lines.push(passed && !warned
+        ? "\n✓ EvalGate gate PASSED"
+        : warned
+            ? `\n⚠ EvalGate gate WARNED: ${failReason ?? report.reasonCode}`
+            : `\n✗ EvalGate gate FAILED: ${failReason ?? report.reasonCode}`);
+    const deltaStr = report.baselineScore != null && report.delta != null
+        ? ` (baseline ${report.baselineScore}, ${report.delta >= 0 ? "+" : ""}${report.delta} pts)`
+        : "";
+    lines.push(`Score: ${report.score ?? 0}/100${deltaStr}`);
+    const failedCases = report.failedCases ?? [];
+    if (failedCases.length > 0) {
+        const toShow = failedCases.slice(0, TOP_N);
+        lines.push(`${failedCases.length} failing case${failedCases.length === 1 ? "" : "s"}:`);
+        for (const fc of toShow) {
+            const label = fc.name ?? fc.input ?? "(unnamed)";
+            const exp = (0, snippet_1.truncateSnippet)(fc.expectedOutput ?? fc.expectedSnippet, 50);
+            const out = (0, snippet_1.truncateSnippet)(fc.output ?? fc.outputSnippet, 50);
+            const reason = out ? `got "${out}"` : "no output";
+            lines.push(`  - "${(0, snippet_1.truncateSnippet)(label, 50)}" → expected: ${exp || "(unknown)"}, ${reason}`);
+        }
+        if (failedCases.length > toShow.length) {
+            lines.push(`  + ${failedCases.length - toShow.length} more`);
+        }
+    }
+    if (report.dashboardUrl) {
+        lines.push(`Dashboard: ${report.dashboardUrl}`);
+    }
+    if (!passed || warned) {
+        lines.push("Next: View full report above, fix failing cases, or adjust gate with --minScore / --maxDrop / --warnDrop");
+    }
+    if (report.explain &&
+        (report.breakdown01 ||
+            report.contribPts ||
+            report.flags?.length ||
+            report.policyEvidence)) {
+        lines.push("");
+        lines.push("--- Explain ---");
+        if (report.contribPts) {
+            const cp = report.contribPts;
+            const pts = [];
+            if (cp.passRatePts != null)
+                pts.push(`passRate: ${cp.passRatePts}`);
+            if (cp.safetyPts != null)
+                pts.push(`safety: ${cp.safetyPts}`);
+            if (cp.compliancePts != null)
+                pts.push(`compliance: ${cp.compliancePts}`);
+            if (cp.performancePts != null)
+                pts.push(`performance: ${cp.performancePts}`);
+            if (pts.length)
+                lines.push(`Contrib pts: ${pts.join(", ")}`);
+        }
+        if (report.breakdown01) {
+            const b = report.breakdown01;
+            const parts = [];
+            if (b.passRate != null)
+                parts.push(`passRate=${b.passRate}`);
+            if (b.safety != null)
+                parts.push(`safety=${b.safety}`);
+            if (b.judge != null)
+                parts.push(`judge=${b.judge}`);
+            if (b.schema != null)
+                parts.push(`schema=${b.schema}`);
+            if (b.latency != null)
+                parts.push(`latency=${b.latency}`);
+            if (b.cost != null)
+                parts.push(`cost=${b.cost}`);
+            if (parts.length)
+                lines.push(`Breakdown: ${parts.join(", ")}`);
+        }
+        if (report.flags && report.flags.length > 0) {
+            lines.push(`Flags: ${report.flags.join(", ")}`);
+        }
+        if (report.thresholds) {
+            const t = report.thresholds;
+            const parts = [];
+            if (t.minScore != null)
+                parts.push(`minScore=${t.minScore}`);
+            if (t.maxDrop != null)
+                parts.push(`maxDrop=${t.maxDrop}`);
+            if (t.minN != null)
+                parts.push(`minN=${t.minN}`);
+            if (parts.length)
+                lines.push(`Thresholds: ${parts.join(", ")}`);
+        }
+        if (report.policyEvidence) {
+            const pe = report.policyEvidence;
+            lines.push(`Policy sub-check failed: ${pe.failedCheck ?? "unknown"}`);
+            if (pe.remediation)
+                lines.push(`Remediation: ${pe.remediation}`);
+            if (pe.snapshot && Object.keys(pe.snapshot).length > 0) {
+                lines.push(`Snapshot: ${JSON.stringify(pe.snapshot)}`);
+            }
+        }
+    }
+    return lines.join("\n");
+}

package/dist/cli/formatters/json.d.ts

@@ -0,0 +1,6 @@
+/**
+ * JSON formatter for evalgate check.
+ * Outputs only JSON, no extra logs.
+ */
+import type { CheckReport } from "./types";
+export declare function formatJson(report: CheckReport): string;

package/dist/cli/formatters/json.js

@@ -0,0 +1,10 @@
+"use strict";
+/**
+ * JSON formatter for evalgate check.
+ * Outputs only JSON, no extra logs.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatJson = formatJson;
+function formatJson(report) {
+    return JSON.stringify(report, null, 0);
+}

package/dist/cli/formatters/pr-comment.d.ts

@@ -0,0 +1,12 @@
+/**
+ * PR comment markdown builder for evalgate check --pr-comment-out.
+ * Produces deterministic markdown for GitHub Action to post as PR comment.
+ */
+import type { CheckReport } from "./types";
+/**
+ * Hidden marker for GitHub Action to find and update existing comment (sticky update).
+ * Action should: 1) post body from file 2) search PR comments for this marker 3) update if found, else create.
+ * Export for use in Action scripts.
+ */
+export declare const PR_COMMENT_MARKER = "<!-- evalgate-gate-comment -->";
+export declare function buildPrComment(report: CheckReport): string;

package/dist/cli/formatters/pr-comment.js

@@ -0,0 +1,103 @@
+"use strict";
+/**
+ * PR comment markdown builder for evalgate check --pr-comment-out.
+ * Produces deterministic markdown for GitHub Action to post as PR comment.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PR_COMMENT_MARKER = void 0;
+exports.buildPrComment = buildPrComment;
+const snippet_1 = require("../render/snippet");
+const TOP_FAILURES = 3;
+function escapeMarkdown(s) {
+    return s.replace(/\|/g, "\\|").replace(/\n/g, " ");
+}
+/**
+ * Hidden marker for GitHub Action to find and update existing comment (sticky update).
+ * Action should: 1) post body from file 2) search PR comments for this marker 3) update if found, else create.
+ * Export for use in Action scripts.
+ */
+exports.PR_COMMENT_MARKER = "<!-- evalgate-gate-comment -->";
+function buildPrComment(report) {
+    const lines = [];
+    lines.push(exports.PR_COMMENT_MARKER);
+    lines.push("");
+    const passed = report.verdict === "pass";
+    const gateApplied = report.gateApplied !== false;
+    // Verdict badge — distinguish "PASS" from "NOT GATED"
+    if (!gateApplied) {
+        lines.push("## ⚠️ EvalGate Regression Gate — NOT APPLIED");
+        lines.push("");
+        lines.push("**Gate not applied: baseline missing.**");
+        if (report.actionableMessage) {
+            lines.push("");
+            lines.push(report.actionableMessage);
+        }
+    }
+    else {
+        lines.push(passed
+            ? "## ✅ EvalGate Regression Gate — PASSED"
+            : "## 🚨 EvalGate Regression Gate — FAILED");
+    }
+    lines.push("");
+    // Score + Delta (skip when gate not applied)
+    const deltaStr = report.baselineScore != null && report.delta != null
+        ? ` (${report.delta >= 0 ? "+" : ""}${report.delta} from baseline ${report.baselineScore})`
+        : "";
+    lines.push(`**Score:** ${report.score ?? 0}/100${deltaStr}`);
+    lines.push("");
+    // ReasonCode
+    lines.push(`**Reason:** ${report.reasonCode}`);
+    if (report.reasonMessage) {
+        lines.push(`_${escapeMarkdown(report.reasonMessage)}_`);
+    }
+    lines.push("");
+    // Policy (if unknown)
+    if (report.policy) {
+        lines.push(`**Policy:** ${report.policy}`);
+        lines.push("");
+    }
+    // Top failures (max 3)
+    const failedCases = report.failedCases ?? [];
+    if (failedCases.length > 0) {
+        lines.push("### Top Issues");
+        lines.push("");
+        for (const fc of failedCases.slice(0, TOP_FAILURES)) {
+            const label = fc.name ?? fc.input ?? "(unnamed)";
+            const reason = fc.reason ?? fc.outputSnippet ?? fc.output ?? "no output";
+            lines.push(`- **${(0, snippet_1.truncateSnippet)(escapeMarkdown(label), 60)}** — ${(0, snippet_1.truncateSnippet)(escapeMarkdown(reason), 80)}`);
+        }
+        if (failedCases.length > TOP_FAILURES) {
+            lines.push(`- _+ ${failedCases.length - TOP_FAILURES} more_`);
+        }
+        lines.push("");
+    }
+    // Explain summary (if --explain)
+    if (report.explain && report.contribPts) {
+        const pts = report.contribPts;
+        const parts = [];
+        if (pts.passRatePts != null)
+            parts.push(`pass rate: ${pts.passRatePts} pts`);
+        if (pts.safetyPts != null)
+            parts.push(`safety: ${pts.safetyPts} pts`);
+        if (pts.compliancePts != null)
+            parts.push(`compliance: ${pts.compliancePts} pts`);
+        if (pts.performancePts != null)
+            parts.push(`performance: ${pts.performancePts} pts`);
+        if (parts.length > 0) {
+            lines.push("### Breakdown");
+            lines.push("");
+            lines.push(parts.join(" | "));
+            lines.push("");
+        }
+    }
+    // Dashboard URL
+    if (report.dashboardUrl) {
+        lines.push(`🔎 [Dashboard](${report.dashboardUrl})`);
+    }
+    // Share URL (if exists)
+    if (report.shareUrl) {
+        lines.push(`🔗 [Share Snapshot](${report.shareUrl})`);
+    }
+    lines.push("");
+    return lines.join("\n");
+}

package/dist/cli/formatters/types.d.ts

@@ -0,0 +1,103 @@
+/**
+ * CheckReport and related types for formatters.
+ */
+export type GateVerdict = "pass" | "warn" | "fail";
+/** "neutral" = exit 0 but gate not applied (e.g. baseline missing with --baseline auto) */
+export type GateMode = "enforced" | "neutral";
+/** Canonical reason codes. Import REASON_CODES from ../reason-codes for constants. */
+export type FailureReasonCode = "PASS" | "WARN_REGRESSION" | "LOW_SAMPLE_SIZE" | "BASELINE_MISSING" | "SCORE_TOO_LOW" | "DELTA_TOO_HIGH" | "COST_BUDGET_EXCEEDED" | "LATENCY_BUDGET_EXCEEDED" | "POLICY_FAILED" | "UNKNOWN" | "LOW_SCORE" | "LOW_PASS_RATE" | "SAFETY_RISK" | "LATENCY_RISK" | "COST_RISK" | "MAX_DROP_EXCEEDED" | "INSUFFICIENT_EVIDENCE" | "POLICY_VIOLATION";
+export type ScoreBreakdown01 = {
+    passRate?: number;
+    safety?: number;
+    judge?: number;
+    schema?: number;
+    latency?: number;
+    cost?: number;
+};
+export type ScoreContribPts = {
+    passRatePts?: number;
+    safetyPts?: number;
+    compliancePts?: number;
+    performancePts?: number;
+};
+export type GateThresholds = {
+    minScore?: number;
+    minPassRate?: number;
+    minSafety?: number;
+    maxDrop?: number;
+    warnDrop?: number;
+    minN?: number;
+    allowWeakEvidence?: boolean;
+    baseline?: "published" | "previous" | "production" | "auto";
+    maxCostUsd?: number;
+    maxLatencyMs?: number;
+    maxCostDeltaUsd?: number;
+};
+export type FailedCase = {
+    testCaseId?: number;
+    status?: "failed" | "error" | "skipped" | "passed";
+    name?: string;
+    input?: string;
+    inputSnippet?: string;
+    expectedOutput?: string;
+    expectedSnippet?: string;
+    output?: string;
+    outputSnippet?: string;
+    reason?: string;
+};
+export type CiContext = {
+    provider?: "github" | "gitlab" | "circle" | "unknown";
+    repo?: string;
+    sha?: string;
+    branch?: string;
+    pr?: number;
+    runUrl?: string;
+    actor?: string;
+};
+/** Current schema version for CheckReport (.evalgate/last-report.json). Bump on breaking changes. */
+export declare const CHECK_REPORT_SCHEMA_VERSION = 1;
+export type CheckReport = {
+    schemaVersion?: number;
+    evaluationId: string;
+    runId?: number;
+    verdict: GateVerdict;
+    /** false when gate not applied (e.g. baseline missing, exit 0) — prevents false confidence */
+    gateApplied: boolean;
+    /** "enforced" = gate ran; "neutral" = exit 0, gate skipped */
+    gateMode: GateMode;
+    reasonCode: FailureReasonCode;
+    /** Actionable message for PR comment / UX */
+    actionableMessage?: string;
+    reasonMessage?: string;
+    score?: number;
+    baselineScore?: number;
+    delta?: number;
+    passRate?: number;
+    safetyPassRate?: number;
+    flags?: string[];
+    breakdown01?: ScoreBreakdown01;
+    contribPts?: ScoreContribPts;
+    thresholds?: GateThresholds;
+    n?: number;
+    evidenceLevel?: "strong" | "medium" | "weak";
+    baselineMissing?: boolean;
+    baselineStatus?: "found" | "missing";
+    dashboardUrl?: string;
+    failedCases?: FailedCase[];
+    failedCasesShown?: number;
+    failedCasesMore?: number;
+    requestId?: string;
+    durationMs?: number;
+    ci?: CiContext;
+    explain?: boolean;
+    shareUrl?: string;
+    policy?: string;
+    baselineRunId?: number;
+    ciRunUrl?: string;
+    /** When --explain and policy failed: which sub-check failed, remediation, snapshot */
+    policyEvidence?: {
+        failedCheck?: string;
+        remediation?: string;
+        snapshot?: Record<string, unknown>;
+    };
+};

package/dist/cli/formatters/types.js

@@ -0,0 +1,8 @@
+"use strict";
+/**
+ * CheckReport and related types for formatters.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CHECK_REPORT_SCHEMA_VERSION = void 0;
+/** Current schema version for CheckReport (.evalgate/last-report.json). Bump on breaking changes. */
+exports.CHECK_REPORT_SCHEMA_VERSION = 1;

package/dist/cli/gate.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Pure gate evaluation. No console output.
+ * Baseline missing → configuration failure (BAD_ARGS), not API_ERROR.
+ */
+import type { QualityLatestData } from "./api";
+import type { CheckArgs } from "./check";
+export type GateResult = {
+    exitCode: number;
+    passed: boolean;
+    reasonCode: string;
+    reasonMessage: string | null;
+    /** true when gate was skipped (e.g. baseline missing + auto) */
+    gateSkipped?: boolean;
+    /** When policy failed: sub-check, remediation, snapshot for explain */
+    policyEvidence?: {
+        failedCheck: string;
+        remediation: string;
+        snapshot?: Record<string, unknown>;
+    };
+};
+export declare function evaluateGate(args: CheckArgs, quality: QualityLatestData): GateResult;