@evalgate/sdk 2.0.0

package/dist/cli/ci-context.js

@@ -0,0 +1,112 @@
+"use strict";
+/**
+ * CI context capture and idempotency key for --onFail import.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.captureCiContext = captureCiContext;
+exports.computeIdempotencyKey = computeIdempotencyKey;
+const node_crypto_1 = require("node:crypto");
+const fs = __importStar(require("node:fs"));
+function readPrFromEventPath() {
+    const path = process.env.GITHUB_EVENT_PATH;
+    if (!path)
+        return undefined;
+    try {
+        const raw = fs.readFileSync(path, "utf8");
+        const event = JSON.parse(raw);
+        return event.pull_request?.number;
+    }
+    catch {
+        return undefined;
+    }
+}
+function readPrFromRef() {
+    const ref = process.env.GITHUB_REF;
+    if (!ref)
+        return undefined;
+    const m = ref.match(/^refs\/pull\/(\d+)\/merge$/);
+    return m ? parseInt(m[1], 10) : undefined;
+}
+function captureCiContext() {
+    const repo = process.env.GITHUB_REPOSITORY;
+    const sha = process.env.GITHUB_SHA;
+    const ref = process.env.GITHUB_REF;
+    const runId = process.env.GITHUB_RUN_ID;
+    const _workflow = process.env.GITHUB_WORKFLOW;
+    const _job = process.env.GITHUB_JOB;
+    const actor = process.env.GITHUB_ACTOR;
+    if (!repo && !sha)
+        return undefined;
+    let provider = "unknown";
+    if (process.env.GITHUB_ACTIONS)
+        provider = "github";
+    else if (process.env.GITLAB_CI)
+        provider = "gitlab";
+    else if (process.env.CIRCLECI)
+        provider = "circle";
+    let runUrl;
+    if (repo && runId) {
+        runUrl = `https://github.com/${repo}/actions/runs/${runId}`;
+    }
+    let pr;
+    if (process.env.GITHUB_EVENT_NAME === "pull_request") {
+        pr = readPrFromEventPath() ?? readPrFromRef();
+    }
+    return {
+        provider,
+        repo,
+        sha,
+        branch: ref?.startsWith("refs/heads/")
+            ? ref.slice("refs/heads/".length)
+            : ref,
+        runUrl,
+        actor,
+        pr,
+    };
+}
+function computeIdempotencyKey(evaluationId, ci) {
+    const repo = ci.repo ?? process.env.GITHUB_REPOSITORY;
+    const workflow = process.env.GITHUB_WORKFLOW ?? "";
+    const job = process.env.GITHUB_JOB ?? "";
+    const sha = ci.sha ?? process.env.GITHUB_SHA ?? "";
+    if (!repo || !sha)
+        return undefined;
+    const input = `${repo}.${workflow}.${job}.${sha}.${evaluationId}`;
+    return hashSha256(input);
+}
+function hashSha256(input) {
+    return (0, node_crypto_1.createHash)("sha256").update(input, "utf8").digest("hex");
+}

package/dist/cli/ci.d.ts

@@ -0,0 +1,45 @@
+/**
+ * UX-401: One-command CI loop (evalgate ci)
+ *
+ * Provides a single command teams put in .github/workflows/* and never think about again.
+ */
+import type { DiffResult } from "./diff";
+import type { RunResult } from "./run";
+/**
+ * CI command options
+ */
+export interface CIOptions {
+    /** Base reference for diff comparison */
+    base?: string;
+    /** Run only impacted specs */
+    impactedOnly?: boolean;
+    /** Output format */
+    format?: "human" | "json" | "github";
+    /** Write run results */
+    writeResults?: boolean;
+}
+/**
+ * CI execution result
+ */
+export interface CIResult {
+    /** Success status */
+    success: boolean;
+    /** Exit code */
+    exitCode: number;
+    /** Execution narrative */
+    narrative: string;
+    /** Run result (if executed) */
+    runResult?: RunResult;
+    /** Diff result (if executed) */
+    diffResult?: DiffResult;
+    /** Error message (if failed) */
+    error?: string;
+}
+/**
+ * Run CI command
+ */
+export declare function runCI(options: CIOptions, projectRoot?: string): Promise<CIResult>;
+/**
+ * CLI entry point
+ */
+export declare function runCICLI(options: CIOptions): Promise<void>;

package/dist/cli/ci.js

@@ -0,0 +1,192 @@
+"use strict";
+/**
+ * UX-401: One-command CI loop (evalgate ci)
+ *
+ * Provides a single command teams put in .github/workflows/* and never think about again.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runCI = runCI;
+exports.runCICLI = runCICLI;
+const fs = __importStar(require("node:fs/promises"));
+const diff_1 = require("./diff");
+const discover_1 = require("./discover");
+const impact_analysis_1 = require("./impact-analysis");
+const run_1 = require("./run");
+const workspace_1 = require("./workspace");
+/**
+ * Run CI command
+ */
+async function runCI(options, projectRoot = process.cwd()) {
+    const workspace = (0, workspace_1.resolveEvalWorkspace)(projectRoot);
+    const narrative = [];
+    try {
+        // 1. Ensure .evalgate workspace exists
+        await fs.mkdir(workspace.evalDir, { recursive: true });
+        narrative.push("✅ workspace ok");
+        // 2. Ensure manifest exists (build if missing)
+        let manifestExists = true;
+        try {
+            await fs.access(workspace.manifestPath);
+        }
+        catch {
+            manifestExists = false;
+        }
+        if (!manifestExists) {
+            console.log("📋 Building evaluation manifest...");
+            await (0, discover_1.discoverSpecs)({ manifest: true });
+            narrative.push("→ manifest built");
+        }
+        else {
+            narrative.push("→ manifest ok");
+        }
+        // 3. Run impact analysis if --impacted-only
+        let impactedSpecCount;
+        if (options.impactedOnly) {
+            const impactResult = await (0, impact_analysis_1.runImpactAnalysis)({
+                baseBranch: options.base || "main",
+            }, projectRoot);
+            impactedSpecCount = impactResult.metadata.impactedCount;
+            narrative.push(`→ impacted specs ${impactedSpecCount}`);
+        }
+        else {
+            narrative.push("→ running all specs");
+        }
+        // 4. Run evaluations
+        const runResult = await (0, run_1.runEvaluations)({
+            impactedOnly: options.impactedOnly,
+            baseBranch: options.base,
+            writeResults: options.writeResults ?? true, // Always write results for CI
+        }, projectRoot);
+        narrative.push(`→ runId ${runResult.runId}`);
+        // 5. Run diff if --base provided
+        let diffResult;
+        if (options.base) {
+            diffResult = await (0, diff_1.runDiff)({
+                base: options.base,
+                head: "last",
+            });
+            if (diffResult.summary.regressions > 0) {
+                narrative.push(`→ diff ${diffResult.summary.regressions} regressions`);
+                return {
+                    success: false,
+                    exitCode: 1,
+                    narrative: narrative.join(" "),
+                    runResult,
+                    diffResult,
+                };
+            }
+            else {
+                narrative.push("→ diff clean");
+            }
+        }
+        else {
+            narrative.push("→ no diff");
+        }
+        // 6. Check for run failures
+        if (runResult.summary.failed > 0) {
+            return {
+                success: false,
+                exitCode: 1,
+                narrative: narrative.join(" "),
+                runResult,
+                diffResult,
+            };
+        }
+        return {
+            success: true,
+            exitCode: 0,
+            narrative: narrative.join(" "),
+            runResult,
+            diffResult,
+        };
+    }
+    catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        // Print next step for debugging
+        printNextStep(errorMessage, options, workspace);
+        return {
+            success: false,
+            exitCode: 2, // Config/infra issue
+            narrative: narrative.join(" "),
+            error: errorMessage,
+        };
+    }
+}
+/**
+ * Print copy/paste debug flow
+ */
+function printNextStep(error, options, workspace) {
+    console.log("\n🔧 Next step for debugging:");
+    if (error.includes("No evaluation manifest found")) {
+        console.log("   evalgate discover --manifest");
+    }
+    else if (error.includes("Base run report not found in CI environment")) {
+        console.log(`   Download base artifact and run: evalgate diff --base .evalgate/base-run.json --head ${workspace.lastRunPath}`);
+    }
+    else if (options.base && error.includes("Base run report not found")) {
+        console.log(`   evalgate explain --report ${workspace.lastRunPath}`);
+    }
+    else {
+        console.log(`   evalgate explain --report ${workspace.lastRunPath}`);
+    }
+    console.log(`   Artifacts: ${workspace.runsDir}/`);
+}
+/**
+ * CLI entry point
+ */
+async function runCICLI(options) {
+    const result = await runCI(options);
+    // Print narrative
+    console.log(`🤖 ${result.narrative}`);
+    // Print detailed results if not clean
+    if (!result.success && result.runResult) {
+        console.log("\n📊 Run Results:");
+        console.log(`   ✅ Passed: ${result.runResult.summary.passed}`);
+        console.log(`   ❌ Failed: ${result.runResult.summary.failed}`);
+        console.log(`   📊 Pass Rate: ${(result.runResult.summary.passRate * 100).toFixed(1)}%`);
+    }
+    if (!result.success && result.diffResult) {
+        console.log("\n🔄 Diff Results:");
+        console.log(`   📉 Regressions: ${result.diffResult.summary.regressions}`);
+        console.log(`   📈 Improvements: ${result.diffResult.summary.improvements}`);
+        console.log(`   📊 Pass Rate Delta: ${(result.diffResult.summary.passRateDelta * 100).toFixed(1)}%`);
+    }
+    if (result.error) {
+        console.log(`\n❌ Error: ${result.error}`);
+    }
+    // Exit with appropriate code
+    process.exit(result.exitCode);
+}

package/dist/cli/config.d.ts

@@ -0,0 +1,30 @@
+/**
+ * EvalGate config loader
+ * Discovery: evalgate.config.json → evalgate.config.js → evalgate.config.cjs → package.json evalgate
+ */
+import { type ProfileName } from "./profiles";
+export interface EvalAIConfig {
+    evaluationId?: string;
+    baseUrl?: string;
+    minScore?: number;
+    minN?: number;
+    maxDrop?: number;
+    warnDrop?: number;
+    allowWeakEvidence?: boolean;
+    baseline?: "published" | "previous" | "production" | "auto";
+    profile?: ProfileName;
+    /** Monorepo: package path → config. Key = path relative to config dir (e.g. "apps/web", "packages/api"). */
+    packages?: Record<string, Partial<EvalAIConfig>>;
+}
+/**
+ * Find config file path in directory, walking up to root
+ */
+export declare function findConfigPath(cwd?: string): string | null;
+/**
+ * Load config from file system
+ */
+export declare function loadConfig(cwd?: string): EvalAIConfig | null;
+/**
+ * Merge config with CLI args. Priority: args > profile > config > defaults.
+ */
+export declare function mergeConfigWithArgs(config: EvalAIConfig | null, args: Partial<Record<string, string | number | boolean>>): Partial<EvalAIConfig>;

package/dist/cli/config.js

@@ -0,0 +1,230 @@
+"use strict";
+/**
+ * EvalGate config loader
+ * Discovery: evalgate.config.json → evalgate.config.js → evalgate.config.cjs → package.json evalgate
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findConfigPath = findConfigPath;
+exports.loadConfig = loadConfig;
+exports.mergeConfigWithArgs = mergeConfigWithArgs;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const profiles_1 = require("./profiles");
+const CONFIG_FILES = [
+    "evalgate.config.json",
+    "evalgate.config.js",
+    "evalgate.config.cjs",
+    "evalai.config.json",
+    "evalai.config.js",
+    "evalai.config.cjs",
+];
+/**
+ * Find config file path in directory, walking up to root
+ */
+function findConfigPath(cwd = process.cwd()) {
+    let dir = path.resolve(cwd);
+    const root = path.parse(dir).root;
+    while (dir !== root) {
+        for (const file of CONFIG_FILES) {
+            const filePath = path.join(dir, file);
+            if (fs.existsSync(filePath)) {
+                return filePath;
+            }
+        }
+        // Check package.json for evalgate or evalai field
+        const pkgPath = path.join(dir, "package.json");
+        if (fs.existsSync(pkgPath)) {
+            try {
+                const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
+                if (pkg.evalgate != null || pkg.evalai != null) {
+                    return pkgPath;
+                }
+            }
+            catch {
+                // ignore
+            }
+        }
+        dir = path.dirname(dir);
+    }
+    return null;
+}
+/**
+ * Load config from file system
+ */
+function loadConfig(cwd = process.cwd()) {
+    const configPath = findConfigPath(cwd);
+    if (!configPath)
+        return null;
+    try {
+        let config = null;
+        if (configPath.endsWith("package.json")) {
+            const pkg = JSON.parse(fs.readFileSync(configPath, "utf-8"));
+            config = (pkg.evalgate ?? pkg.evalai);
+        }
+        else {
+            const content = fs.readFileSync(configPath, "utf-8");
+            if (configPath.endsWith(".json")) {
+                config = JSON.parse(content);
+            }
+            else if (configPath.endsWith(".js") || configPath.endsWith(".cjs")) {
+                try {
+                    config = JSON.parse(content);
+                }
+                catch {
+                    return null;
+                }
+            }
+        }
+        if (!config)
+            return null;
+        if (config.packages && Object.keys(config.packages).length > 0) {
+            const configDir = path.dirname(configPath);
+            const rel = path.relative(configDir, path.resolve(cwd));
+            const relNorm = rel.split(path.sep).join("/");
+            const pkgConfig = config.packages[relNorm];
+            if (pkgConfig) {
+                return { ...config, ...pkgConfig, packages: config.packages };
+            }
+            for (const key of Object.keys(config.packages)) {
+                if (relNorm === key || relNorm.startsWith(`${key}/`)) {
+                    return {
+                        ...config,
+                        ...config.packages[key],
+                        packages: config.packages,
+                    };
+                }
+            }
+        }
+        return config;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Merge config with CLI args. Priority: args > profile > config > defaults.
+ */
+function mergeConfigWithArgs(config, args) {
+    const merged = {};
+    if (config) {
+        if (config.evaluationId)
+            merged.evaluationId = config.evaluationId;
+        if (config.baseUrl)
+            merged.baseUrl = config.baseUrl;
+        if (config.minScore != null)
+            merged.minScore = config.minScore;
+        if (config.minN != null)
+            merged.minN = config.minN;
+        if (config.maxDrop != null)
+            merged.maxDrop = config.maxDrop;
+        if (config.warnDrop != null)
+            merged.warnDrop = config.warnDrop;
+        if (config.allowWeakEvidence != null)
+            merged.allowWeakEvidence = config.allowWeakEvidence;
+        if (config.baseline)
+            merged.baseline = config.baseline;
+        if (config.profile)
+            merged.profile = config.profile;
+    }
+    // Profile defaults (from --profile or config.profile). Apply before args override.
+    const profileName = (args.profile ?? merged.profile);
+    if (profileName && profileName in profiles_1.PROFILES) {
+        const profile = profiles_1.PROFILES[profileName];
+        if (merged.minScore === undefined && args.minScore === undefined)
+            merged.minScore = profile.minScore;
+        if (merged.maxDrop === undefined && args.maxDrop === undefined)
+            merged.maxDrop = profile.maxDrop;
+        if (merged.warnDrop === undefined &&
+            args.warnDrop === undefined &&
+            "warnDrop" in profile)
+            merged.warnDrop = profile.warnDrop;
+        if (merged.minN === undefined && args.minN === undefined)
+            merged.minN = profile.minN;
+        if (merged.allowWeakEvidence === undefined &&
+            args.allowWeakEvidence === undefined)
+            merged.allowWeakEvidence = profile.allowWeakEvidence;
+    }
+    // Args override
+    if (args.evaluationId !== undefined && args.evaluationId !== "") {
+        merged.evaluationId = String(args.evaluationId);
+    }
+    if (args.baseUrl !== undefined && args.baseUrl !== "") {
+        merged.baseUrl = String(args.baseUrl);
+    }
+    if (args.minScore !== undefined) {
+        merged.minScore =
+            typeof args.minScore === "number"
+                ? args.minScore
+                : parseInt(String(args.minScore), 10);
+    }
+    if (args.maxDrop !== undefined) {
+        merged.maxDrop =
+            typeof args.maxDrop === "number"
+                ? args.maxDrop
+                : parseInt(String(args.maxDrop), 10);
+    }
+    if (args.warnDrop !== undefined) {
+        merged.warnDrop =
+            typeof args.warnDrop === "number"
+                ? args.warnDrop
+                : parseInt(String(args.warnDrop), 10);
+    }
+    if (args.minN !== undefined) {
+        merged.minN =
+            typeof args.minN === "number"
+                ? args.minN
+                : parseInt(String(args.minN), 10);
+    }
+    if (args.allowWeakEvidence !== undefined) {
+        merged.allowWeakEvidence =
+            args.allowWeakEvidence === true ||
+                args.allowWeakEvidence === "true" ||
+                args.allowWeakEvidence === "1";
+    }
+    if (args.baseline !== undefined && args.baseline !== "") {
+        const b = String(args.baseline);
+        if (b === "auto" || b === "previous" || b === "production") {
+            merged.baseline = b;
+        }
+        else {
+            merged.baseline = "published";
+        }
+    }
+    if (args.profile !== undefined && args.profile !== "") {
+        merged.profile = String(args.profile);
+    }
+    return merged;
+}

package/dist/cli/constants.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Standardized exit codes for evalgate check.
+ */
+export declare const EXIT: {
+    readonly PASS: 0;
+    readonly SCORE_BELOW: 1;
+    readonly REGRESSION: 2;
+    readonly POLICY_VIOLATION: 3;
+    readonly API_ERROR: 4;
+    readonly BAD_ARGS: 5;
+    readonly LOW_N: 6;
+    readonly WEAK_EVIDENCE: 7;
+    /** Near-regression: score dropped within warn band (warnDrop ≤ drop < maxDrop) */
+    readonly WARN_REGRESSION: 8;
+};

package/dist/cli/constants.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EXIT = void 0;
+/**
+ * Standardized exit codes for evalgate check.
+ */
+exports.EXIT = {
+    PASS: 0,
+    SCORE_BELOW: 1,
+    REGRESSION: 2,
+    POLICY_VIOLATION: 3,
+    API_ERROR: 4,
+    BAD_ARGS: 5,
+    LOW_N: 6,
+    WEAK_EVIDENCE: 7,
+    /** Near-regression: score dropped within warn band (warnDrop ≤ drop < maxDrop) */
+    WARN_REGRESSION: 8,
+};