@evalgate/sdk 2.0.0

package/dist/cli/regression-gate.js

@@ -0,0 +1,341 @@
+"use strict";
+/**
+ * evalgate gate — Run the regression gate
+ *
+ * Two modes:
+ *   1. Project mode: delegates to eval:regression-gate npm script (full gate)
+ *   2. Built-in mode: runs `npm test`, compares against evals/baseline.json
+ *
+ * Built-in mode activates when no eval:regression-gate script is defined,
+ * making `npx evalgate gate` work for any project after `npx evalgate init`.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseGateArgs = parseGateArgs;
+exports.runGate = runGate;
+const node_child_process_1 = require("node:child_process");
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const REPORT_REL = "evals/regression-report.json";
+const BASELINE_REL = "evals/baseline.json";
+/** Detect the package manager used in the project */
+function detectPackageManager(cwd) {
+    if (fs.existsSync(path.join(cwd, "pnpm-lock.yaml")))
+        return "pnpm";
+    if (fs.existsSync(path.join(cwd, "yarn.lock")))
+        return "yarn";
+    return "npm";
+}
+function parseGateArgs(argv) {
+    const args = { format: "human" };
+    for (let i = 0; i < argv.length; i++) {
+        if (argv[i] === "--format" && argv[i + 1]) {
+            const fmt = argv[i + 1];
+            if (fmt === "json" || fmt === "github" || fmt === "human") {
+                args.format = fmt;
+            }
+            i++;
+        }
+    }
+    return args;
+}
+function detectRunner(cwd) {
+    const pkgPath = path.join(cwd, "package.json");
+    try {
+        const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
+        const testCmd = pkg.scripts?.test ?? "";
+        if (testCmd.includes("vitest"))
+            return "vitest";
+        if (testCmd.includes("jest"))
+            return "jest";
+        if (testCmd.includes("mocha"))
+            return "mocha";
+        if (testCmd.includes("node --test"))
+            return "node:test";
+        if (testCmd.includes("ava"))
+            return "ava";
+        if (testCmd.includes("tap"))
+            return "tap";
+    }
+    catch {
+        // ignore
+    }
+    return "unknown";
+}
+function runBuiltinGate(cwd) {
+    const t0 = Date.now();
+    const baselinePath = path.join(cwd, BASELINE_REL);
+    const now = new Date().toISOString();
+    const pm = detectPackageManager(cwd);
+    const command = `${pm} test`;
+    const runner = detectRunner(cwd);
+    // Load baseline
+    if (!fs.existsSync(baselinePath)) {
+        return {
+            schemaVersion: 1,
+            timestamp: now,
+            exitCode: 2,
+            category: "infra_error",
+            passed: false,
+            failures: ["Baseline file not found. Run: npx evalgate init"],
+            deltas: [],
+            baseline: null,
+            durationMs: Date.now() - t0,
+            command,
+            runner,
+        };
+    }
+    let baselineData;
+    try {
+        baselineData = JSON.parse(fs.readFileSync(baselinePath, "utf-8"));
+    }
+    catch {
+        return {
+            schemaVersion: 1,
+            timestamp: now,
+            exitCode: 2,
+            category: "infra_error",
+            passed: false,
+            failures: ["Failed to parse evals/baseline.json"],
+            deltas: [],
+            baseline: null,
+            durationMs: Date.now() - t0,
+            command,
+            runner,
+        };
+    }
+    const baselineMeta = baselineData.updatedAt
+        ? {
+            updatedAt: baselineData.updatedAt,
+            updatedBy: baselineData.updatedBy ?? "unknown",
+        }
+        : null;
+    // Run tests
+    const isWin = process.platform === "win32";
+    const result = (0, node_child_process_1.spawnSync)(pm, ["test"], {
+        cwd,
+        stdio: "pipe",
+        shell: isWin,
+        timeout: 300000,
+    });
+    const testsPassed = result.status === 0;
+    const output = (result.stdout?.toString() ?? "") + (result.stderr?.toString() ?? "");
+    // Try to extract test count
+    let testCount = 0;
+    const countMatch = output.match(/(\d+)\s+(?:tests?|specs?)\s+(?:passed|completed)/i) ??
+        output.match(/Tests:\s+(\d+)\s+passed/i) ??
+        output.match(/(\d+)\s+passing/i) ??
+        output.match(/Test Files\s+\d+\s+passed.*\n\s+Tests\s+(\d+)\s+passed/i);
+    if (countMatch)
+        testCount = parseInt(countMatch[1], 10);
+    // Compare against baseline
+    const baselinePassed = baselineData.confidenceTests?.passed ?? true;
+    const baselineTotal = baselineData.confidenceTests?.total ?? 0;
+    const failures = [];
+    const deltas = [];
+    // Delta: tests passing
+    deltas.push({
+        metric: "tests_passing",
+        baseline: baselinePassed,
+        current: testsPassed,
+        delta: testsPassed === baselinePassed ? "0" : testsPassed ? "+1" : "-1",
+        status: testsPassed ? "pass" : "fail",
+    });
+    if (!testsPassed && baselinePassed) {
+        failures.push("Tests were passing in baseline but are now failing");
+    }
+    // Delta: test count (only if we captured counts)
+    if (testCount > 0 || baselineTotal > 0) {
+        const countDelta = testCount - baselineTotal;
+        deltas.push({
+            metric: "test_count",
+            baseline: baselineTotal,
+            current: testCount,
+            delta: countDelta >= 0 ? `+${countDelta}` : `${countDelta}`,
+            status: testCount >= baselineTotal ? "pass" : "fail",
+        });
+        if (testCount < baselineTotal) {
+            failures.push(`Test count dropped from ${baselineTotal} to ${testCount} (${countDelta})`);
+        }
+    }
+    const hasRegression = failures.length > 0;
+    return {
+        schemaVersion: 1,
+        timestamp: now,
+        exitCode: hasRegression ? 1 : 0,
+        category: hasRegression ? "regression" : "pass",
+        passed: !hasRegression,
+        failures,
+        deltas,
+        baseline: baselineMeta,
+        durationMs: Date.now() - t0,
+        command,
+        runner,
+    };
+}
+// ── Format helpers ──
+function formatHuman(report) {
+    const icon = report.passed ? "✅" : "❌";
+    console.log(`\n${icon} EvalGate Gate: ${report.category.toUpperCase()}\n`);
+    if (report.deltas.length > 0) {
+        const pad = (s, n) => s.padEnd(n);
+        console.log(`  ${pad("Metric", 16)} ${pad("Baseline", 10)} ${pad("Current", 10)} ${pad("Delta", 8)} Status`);
+        console.log(`  ${"-".repeat(16)} ${"-".repeat(10)} ${"-".repeat(10)} ${"-".repeat(8)} ------`);
+        for (const d of report.deltas) {
+            const si = d.status === "pass" ? "✔" : "✖";
+            console.log(`  ${pad(d.metric, 16)} ${pad(String(d.baseline), 10)} ${pad(String(d.current), 10)} ${pad(d.delta, 8)} ${si}`);
+        }
+    }
+    if (report.failures.length > 0) {
+        console.log("\n  Failures:");
+        for (const f of report.failures) {
+            console.log(`    • ${f}`);
+        }
+    }
+    console.log("");
+}
+function formatGithub(report) {
+    const icon = report.passed ? "✅" : "❌";
+    const lines = [
+        `## ${icon} EvalGate Gate: ${report.category}`,
+        "",
+        "| Metric | Baseline | Current | Delta | Status |",
+        "|--------|----------|---------|-------|--------|",
+    ];
+    for (const d of report.deltas) {
+        const si = d.status === "pass" ? "✅" : "❌";
+        lines.push(`| ${d.metric} | ${d.baseline} | ${d.current} | ${d.delta} | ${si} |`);
+    }
+    if (report.failures.length > 0) {
+        lines.push("", "### Failures", "");
+        for (const f of report.failures) {
+            lines.push(`- ${f}`);
+        }
+    }
+    lines.push("", `Schema version: ${report.schemaVersion}`);
+    const md = lines.join("\n");
+    // Write to $GITHUB_STEP_SUMMARY if available
+    const summaryPath = process.env.GITHUB_STEP_SUMMARY;
+    if (summaryPath) {
+        try {
+            fs.appendFileSync(summaryPath, `${md}\n`);
+        }
+        catch {
+            // ignore if not writable
+        }
+    }
+    console.log(md);
+}
+function formatReport(report, args) {
+    if (args.format === "json") {
+        process.stdout.write(JSON.stringify(report, null, 2));
+    }
+    else if (args.format === "github") {
+        formatGithub(report);
+    }
+    else {
+        formatHuman(report);
+    }
+}
+// ── Main ──
+function runGate(argv) {
+    const cwd = process.cwd();
+    const args = parseGateArgs(argv);
+    // Check for package.json
+    const pkgPath = path.join(cwd, "package.json");
+    if (!fs.existsSync(pkgPath)) {
+        console.error("❌ No package.json found. Run this from your project root.");
+        return 1;
+    }
+    let pkg;
+    try {
+        pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
+    }
+    catch {
+        console.error("❌ Failed to parse package.json");
+        return 1;
+    }
+    // ── Project mode: delegate to eval:regression-gate if it exists ──
+    if (pkg.scripts?.["eval:regression-gate"]) {
+        const pm = detectPackageManager(cwd);
+        const isWin = process.platform === "win32";
+        const stdio = args.format === "json" ? "pipe" : "inherit";
+        const result = (0, node_child_process_1.spawnSync)(pm, ["run", "eval:regression-gate"], {
+            cwd,
+            stdio: stdio,
+            shell: isWin,
+        });
+        const exitCode = result.status ?? 1;
+        // Post-process report for json/github formats
+        if (args.format === "json") {
+            const reportPath = path.join(cwd, REPORT_REL);
+            if (fs.existsSync(reportPath)) {
+                process.stdout.write(fs.readFileSync(reportPath, "utf-8"));
+            }
+            else {
+                console.error(JSON.stringify({
+                    error: "regression-report.json not found",
+                    exitCode,
+                }));
+            }
+        }
+        else if (args.format === "github") {
+            const reportPath = path.join(cwd, REPORT_REL);
+            if (fs.existsSync(reportPath)) {
+                try {
+                    const report = JSON.parse(fs.readFileSync(reportPath, "utf-8"));
+                    formatGithub(report);
+                }
+                catch {
+                    // human output already printed
+                }
+            }
+        }
+        return exitCode;
+    }
+    // ── Built-in mode: run tests + compare against baseline ──
+    if (args.format === "human") {
+        console.log("\n  Running EvalGate regression gate (built-in mode)...\n");
+    }
+    const report = runBuiltinGate(cwd);
+    // Write report artifact
+    const evalsDir = path.join(cwd, "evals");
+    if (!fs.existsSync(evalsDir)) {
+        fs.mkdirSync(evalsDir, { recursive: true });
+    }
+    fs.writeFileSync(path.join(cwd, REPORT_REL), `${JSON.stringify(report, null, 2)}\n`);
+    formatReport(report, args);
+    return report.exitCode;
+}

package/dist/cli/render/snippet.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Truncate a string for deterministic output.
+ * Replaces newlines with space, caps length.
+ */
+export declare function truncateSnippet(s: string | undefined | null, maxLen?: number): string;

package/dist/cli/render/snippet.js

@@ -0,0 +1,15 @@
+"use strict";
+/**
+ * Truncate a string for deterministic output.
+ * Replaces newlines with space, caps length.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.truncateSnippet = truncateSnippet;
+function truncateSnippet(s, maxLen = 140) {
+    if (s == null)
+        return "";
+    const normalized = s.replace(/\s+/g, " ").trim();
+    if (normalized.length <= maxLen)
+        return normalized;
+    return `${normalized.slice(0, maxLen)}…`;
+}

package/dist/cli/render/sort.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Deterministic ordering for failed cases.
+ * Sort by status severity (failed > error > skipped > passed), then by testCaseId asc.
+ */
+export interface SortableCase {
+    status?: string;
+    testCaseId?: number;
+    [key: string]: unknown;
+}
+export declare function sortFailedCases<T extends SortableCase>(cases: T[]): T[];

package/dist/cli/render/sort.js

@@ -0,0 +1,24 @@
+"use strict";
+/**
+ * Deterministic ordering for failed cases.
+ * Sort by status severity (failed > error > skipped > passed), then by testCaseId asc.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sortFailedCases = sortFailedCases;
+const STATUS_SEVERITY = {
+    failed: 0,
+    error: 1,
+    skipped: 2,
+    passed: 3,
+};
+function sortFailedCases(cases) {
+    return [...cases].sort((a, b) => {
+        const sevA = STATUS_SEVERITY[a.status?.toLowerCase() ?? ""] ?? 4;
+        const sevB = STATUS_SEVERITY[b.status?.toLowerCase() ?? ""] ?? 4;
+        if (sevA !== sevB)
+            return sevA - sevB;
+        const idA = a.testCaseId ?? 0;
+        const idB = b.testCaseId ?? 0;
+        return idA - idB;
+    });
+}

package/dist/cli/report/build-check-report.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Build CheckReport from API data and gate result.
+ * Normalizes failed cases (truncate, sort), dashboard URL, top N + more.
+ */
+import type { QualityLatestData, RunDetailsData } from "../api";
+import type { CheckArgs } from "../check";
+import { type CheckReport } from "../formatters/types";
+import type { GateResult } from "../gate";
+export type BuildReportInput = {
+    args: CheckArgs;
+    quality: QualityLatestData;
+    runDetails?: RunDetailsData | null;
+    gateResult: GateResult;
+    requestId?: string;
+    shareUrl?: string;
+    baselineRunId?: number | null;
+    ciRunUrl?: string | null;
+};
+export declare function buildCheckReport(input: BuildReportInput): CheckReport;

package/dist/cli/report/build-check-report.js

@@ -0,0 +1,132 @@
+"use strict";
+/**
+ * Build CheckReport from API data and gate result.
+ * Normalizes failed cases (truncate, sort), dashboard URL, top N + more.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildCheckReport = buildCheckReport;
+const types_1 = require("../formatters/types");
+const snippet_1 = require("../render/snippet");
+const sort_1 = require("../render/sort");
+const TOP_N = 3;
+/** ContribPts from weights: passRate*50, safety*25, (0.6*judge+0.4*schema)*15, (0.6*latency+0.4*cost)*10 */
+function computeContribPts(b) {
+    const pr = b.passRate ?? 0;
+    const s = b.safety ?? 0;
+    const j = b.judge ?? 0;
+    const sc = b.schema ?? 0;
+    const l = b.latency ?? 0;
+    const c = b.cost ?? 0;
+    return {
+        passRatePts: Math.round(pr * 50 * 10) / 10,
+        safetyPts: Math.round(s * 25 * 10) / 10,
+        compliancePts: Math.round((0.6 * j + 0.4 * sc) * 15 * 10) / 10,
+        performancePts: Math.round((0.6 * l + 0.4 * c) * 10 * 10) / 10,
+    };
+}
+const SNIPPET_MAX = 50;
+function buildCheckReport(input) {
+    const { args, quality, runDetails, gateResult, requestId } = input;
+    const score = quality?.score ?? 0;
+    const total = quality?.total ?? null;
+    const baselineScore = quality?.baselineScore ?? null;
+    const regressionDelta = quality?.regressionDelta ?? null;
+    const evaluationRunId = quality?.evaluationRunId;
+    const breakdown = quality?.breakdown ?? {};
+    const flags = (quality?.flags ?? []);
+    const baseUrl = args.baseUrl.replace(/\/$/, "");
+    const dashboardUrl = evaluationRunId != null
+        ? `${baseUrl}/evaluations/${args.evaluationId}/runs/${evaluationRunId}`
+        : undefined;
+    // Build failed cases from run details
+    let failedCases = [];
+    if (runDetails?.results && evaluationRunId != null) {
+        const raw = runDetails.results
+            .filter((r) => r.status === "failed")
+            .map((r) => ({
+            testCaseId: r.testCaseId,
+            status: "failed",
+            name: r.test_cases?.name,
+            input: r.test_cases?.input,
+            expectedOutput: r.test_cases?.expectedOutput,
+            output: r.output,
+        }));
+        failedCases = (0, sort_1.sortFailedCases)(raw).map((fc) => ({
+            ...fc,
+            inputSnippet: (0, snippet_1.truncateSnippet)(fc.input, SNIPPET_MAX),
+            expectedSnippet: (0, snippet_1.truncateSnippet)(fc.expectedOutput, SNIPPET_MAX),
+            outputSnippet: (0, snippet_1.truncateSnippet)(fc.output, SNIPPET_MAX),
+        }));
+    }
+    const failedCasesShown = Math.min(failedCases.length, TOP_N);
+    const failedCasesMore = failedCases.length - failedCasesShown;
+    const breakdown01 = Object.keys(breakdown).length > 0
+        ? breakdown
+        : undefined;
+    const contribPts = args.explain && breakdown01 ? computeContribPts(breakdown01) : undefined;
+    const gateSkipped = gateResult.gateSkipped === true;
+    const gateApplied = !gateSkipped;
+    const gateMode = gateSkipped ? "neutral" : "enforced";
+    const actionableMessage = gateSkipped
+        ? "Gate not applied: baseline missing. Publish a baseline from the dashboard, or run with --baseline previous once you have runs."
+        : (gateResult.reasonMessage ?? undefined);
+    const verdict = gateResult.reasonCode === "WARN_REGRESSION"
+        ? "warn"
+        : gateResult.passed
+            ? "pass"
+            : "fail";
+    const report = {
+        schemaVersion: types_1.CHECK_REPORT_SCHEMA_VERSION,
+        evaluationId: args.evaluationId,
+        runId: evaluationRunId,
+        verdict,
+        gateApplied,
+        gateMode,
+        actionableMessage,
+        shareUrl: input.shareUrl,
+        policy: args.policy,
+        baselineRunId: input.baselineRunId ?? quality?.baselineRunId ?? undefined,
+        ciRunUrl: input.ciRunUrl ?? undefined,
+        reasonCode: gateResult.reasonCode,
+        reasonMessage: gateResult.reasonMessage ?? undefined,
+        score,
+        baselineScore: baselineScore ?? undefined,
+        delta: regressionDelta ?? undefined,
+        n: total ?? undefined,
+        evidenceLevel: quality?.evidenceLevel ?? undefined,
+        baselineMissing: quality?.baselineMissing === true,
+        baselineStatus: quality?.baselineMissing === true
+            ? "missing"
+            : quality?.baselineScore != null
+                ? "found"
+                : undefined,
+        flags: flags.length > 0 ? [...flags].sort() : undefined,
+        breakdown01,
+        contribPts,
+        thresholds: {
+            minScore: args.minScore,
+            maxDrop: args.maxDrop,
+            warnDrop: args.warnDrop,
+            minN: args.minN,
+            allowWeakEvidence: args.allowWeakEvidence,
+            baseline: args.baseline,
+            maxCostUsd: args.maxCostUsd,
+            maxLatencyMs: args.maxLatencyMs,
+            maxCostDeltaUsd: args.maxCostDeltaUsd,
+        },
+        dashboardUrl,
+        failedCases,
+        failedCasesShown: failedCases.length > 0 ? failedCasesShown : undefined,
+        failedCasesMore: failedCasesMore > 0 ? failedCasesMore : undefined,
+        requestId,
+        explain: args.explain,
+        policyEvidence: args.explain && gateResult.policyEvidence
+            ? {
+                failedCheck: gateResult.policyEvidence.failedCheck,
+                remediation: gateResult.policyEvidence.remediation,
+                snapshot: gateResult.policyEvidence.snapshot,
+            }
+            : undefined,
+    };
+    return report;
+}

package/dist/cli/run.d.ts

@@ -0,0 +1,101 @@
+/**
+ * TICKET 4 — Unified evalgate run CLI Command
+ *
+ * Goal: Consolidated execution interface that consumes manifest
+ *
+ * Features:
+ * - Manifest loading and spec filtering
+ * - --impacted-only integration with impact analysis
+ * - Local executor integration
+ * - .evalgate/last-run.json output
+ * - Legacy mode compatibility
+ */
+/**
+ * Run execution options
+ */
+export interface RunOptions {
+    /** Filter to specific spec IDs */
+    specIds?: string[];
+    /** Run only impacted specs (requires base branch) */
+    impactedOnly?: boolean;
+    /** Base branch for impact analysis */
+    baseBranch?: string;
+    /** Output format */
+    format?: "human" | "json";
+    /** Write run results to file */
+    writeResults?: boolean;
+}
+/**
+ * Run execution result
+ */
+export interface RunResult {
+    /** Schema version for compatibility checking */
+    schemaVersion: number;
+    /** Unique run identifier */
+    runId: string;
+    /** Execution metadata */
+    metadata: {
+        startedAt: number;
+        completedAt: number;
+        duration: number;
+        totalSpecs: number;
+        executedSpecs: number;
+        mode: "spec" | "legacy";
+    };
+    /** Individual spec results */
+    results: SpecResult[];
+    /** Summary statistics */
+    summary: {
+        passed: number;
+        failed: number;
+        skipped: number;
+        passRate: number;
+    };
+}
+/**
+ * Individual spec result
+ */
+export interface SpecResult {
+    /** Spec identifier */
+    specId: string;
+    /** Spec name */
+    name: string;
+    /** File path */
+    filePath: string;
+    /** Execution result */
+    result: {
+        status: "passed" | "failed" | "skipped";
+        score?: number;
+        error?: string;
+        duration: number;
+    };
+}
+/**
+ * Run evaluation specifications
+ */
+export declare function runEvaluations(options: RunOptions, projectRoot?: string): Promise<RunResult>;
+/**
+ * Run index entry
+ */
+export interface RunIndexEntry {
+    runId: string;
+    createdAt: number;
+    gitSha?: string;
+    branch?: string;
+    mode: "spec" | "legacy";
+    specCount: number;
+    passRate: number;
+    avgScore: number;
+}
+/**
+ * Print human-readable results
+ */
+export declare function printHumanResults(result: RunResult): void;
+/**
+ * Print JSON results
+ */
+export declare function printJsonResults(result: RunResult): void;
+/**
+ * CLI entry point
+ */
+export declare function runEvaluationsCLI(options: RunOptions): Promise<void>;