@evalgate/sdk 2.0.0

package/dist/cli/gate.js

@@ -0,0 +1,179 @@
+"use strict";
+/**
+ * Pure gate evaluation. No console output.
+ * Baseline missing → configuration failure (BAD_ARGS), not API_ERROR.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evaluateGate = evaluateGate;
+const constants_1 = require("./constants");
+const policy_packs_1 = require("./policy-packs");
+const reason_codes_1 = require("./reason-codes");
+function evaluateGate(args, quality) {
+    const score = quality?.score ?? 0;
+    const total = quality?.total ?? null;
+    const evidenceLevel = quality?.evidenceLevel ?? null;
+    const _baselineScore = quality?.baselineScore ?? null;
+    const regressionDelta = quality?.regressionDelta ?? null;
+    const baselineMissing = quality?.baselineMissing === true;
+    const breakdown = quality?.breakdown ?? {};
+    const policyFlags = (quality?.flags ?? []);
+    const avgLatencyMs = quality?.avgLatencyMs ?? null;
+    const costUsd = quality?.costUsd ?? null;
+    const baselineCostUsd = quality?.baselineCostUsd ?? null;
+    // Baseline missing FIRST: --baseline auto → exit 0 (neutral, gate not applied); others → BAD_ARGS
+    // Must run before budget gates so baseline missing + maxCostDeltaUsd ⇒ neutral, not budget failure
+    if (baselineMissing) {
+        const msg = args.baseline === "auto"
+            ? "No baseline found. Tip: Publish a baseline from the dashboard, or run with --baseline previous once you have runs."
+            : args.baseline === "production"
+                ? "No prod runs exist for this evaluation. Tag runs with environment=prod before using --baseline production."
+                : `Baseline (${args.baseline}) not found. Ensure a baseline run exists (e.g. published run, previous run, or prod-tagged run).`;
+        if (args.baseline === "auto") {
+            return {
+                exitCode: constants_1.EXIT.PASS,
+                passed: false,
+                reasonCode: reason_codes_1.REASON_CODES.BASELINE_MISSING,
+                reasonMessage: msg,
+                gateSkipped: true,
+            };
+        }
+        if (args.baseline !== "published" || args.maxDrop !== undefined) {
+            return {
+                exitCode: constants_1.EXIT.BAD_ARGS,
+                passed: false,
+                reasonCode: reason_codes_1.REASON_CODES.BASELINE_MISSING,
+                reasonMessage: msg,
+            };
+        }
+    }
+    // Budget gates (after baseline check)
+    if (args.maxCostUsd != null && costUsd != null && costUsd > args.maxCostUsd) {
+        return {
+            exitCode: constants_1.EXIT.SCORE_BELOW,
+            passed: false,
+            reasonCode: reason_codes_1.REASON_CODES.COST_BUDGET_EXCEEDED,
+            reasonMessage: `cost $${costUsd.toFixed(4)} exceeds maxCostUsd $${args.maxCostUsd.toFixed(4)}`,
+        };
+    }
+    if (args.maxLatencyMs != null &&
+        avgLatencyMs != null &&
+        avgLatencyMs > args.maxLatencyMs) {
+        return {
+            exitCode: constants_1.EXIT.SCORE_BELOW,
+            passed: false,
+            reasonCode: reason_codes_1.REASON_CODES.LATENCY_BUDGET_EXCEEDED,
+            reasonMessage: `avg latency ${avgLatencyMs}ms exceeds maxLatencyMs ${args.maxLatencyMs}`,
+        };
+    }
+    if (args.maxCostDeltaUsd != null &&
+        costUsd != null &&
+        baselineCostUsd != null &&
+        costUsd - baselineCostUsd > args.maxCostDeltaUsd) {
+        return {
+            exitCode: constants_1.EXIT.SCORE_BELOW,
+            passed: false,
+            reasonCode: reason_codes_1.REASON_CODES.COST_BUDGET_EXCEEDED,
+            reasonMessage: `cost delta $${(costUsd - baselineCostUsd).toFixed(4)} exceeds maxCostDeltaUsd $${args.maxCostDeltaUsd.toFixed(4)}`,
+        };
+    }
+    // minN gate
+    if (args.minN !== undefined && total !== null && total < args.minN) {
+        return {
+            exitCode: constants_1.EXIT.LOW_N,
+            passed: false,
+            reasonCode: reason_codes_1.REASON_CODES.LOW_SAMPLE_SIZE,
+            reasonMessage: `total test cases (${total}) < minN (${args.minN})`,
+        };
+    }
+    // allowWeakEvidence gate
+    if (!args.allowWeakEvidence && evidenceLevel === "weak") {
+        return {
+            exitCode: constants_1.EXIT.WEAK_EVIDENCE,
+            passed: false,
+            reasonCode: reason_codes_1.REASON_CODES.LOW_SAMPLE_SIZE,
+            reasonMessage: "evidence level is 'weak' (use --allowWeakEvidence to permit)",
+        };
+    }
+    // Compute gate result
+    if (args.minScore > 0 && score < args.minScore) {
+        return {
+            exitCode: constants_1.EXIT.SCORE_BELOW,
+            passed: false,
+            reasonCode: reason_codes_1.REASON_CODES.SCORE_TOO_LOW,
+            reasonMessage: `score ${score} < minScore ${args.minScore}`,
+        };
+    }
+    // warnDrop: soft warning band; maxDrop: hard fail
+    if (args.maxDrop !== undefined &&
+        regressionDelta !== null &&
+        regressionDelta < -args.maxDrop) {
+        return {
+            exitCode: constants_1.EXIT.REGRESSION,
+            passed: false,
+            reasonCode: reason_codes_1.REASON_CODES.DELTA_TOO_HIGH,
+            reasonMessage: `score dropped ${Math.abs(regressionDelta)} pts from baseline (max allowed: ${args.maxDrop})`,
+        };
+    }
+    if (args.warnDrop !== undefined &&
+        regressionDelta !== null &&
+        regressionDelta < -args.warnDrop &&
+        (args.maxDrop === undefined || regressionDelta >= -args.maxDrop)) {
+        return {
+            exitCode: constants_1.EXIT.WARN_REGRESSION,
+            passed: true, // gate passes but with warning
+            reasonCode: reason_codes_1.REASON_CODES.WARN_REGRESSION,
+            reasonMessage: `score dropped ${Math.abs(regressionDelta)} pts from baseline (warn threshold: ${args.warnDrop}${args.maxDrop != null ? `, fail at ${args.maxDrop}` : ""})`,
+        };
+    }
+    if (args.policy) {
+        const pack = (0, policy_packs_1.resolvePolicyPack)(args.policy);
+        if (!pack) {
+            const valid = (0, policy_packs_1.getValidPolicyVersions)().join(", ");
+            return {
+                exitCode: constants_1.EXIT.BAD_ARGS,
+                passed: false,
+                reasonCode: reason_codes_1.REASON_CODES.UNKNOWN,
+                reasonMessage: `Unknown policy or version: ${args.policy}. Valid: ${valid}`,
+            };
+        }
+        const { requiredSafetyRate, maxFlags } = pack.thresholds;
+        const safetyRate = breakdown?.safety ?? 0;
+        if (safetyRate < requiredSafetyRate) {
+            return {
+                exitCode: constants_1.EXIT.POLICY_VIOLATION,
+                passed: false,
+                reasonCode: reason_codes_1.REASON_CODES.POLICY_FAILED,
+                reasonMessage: `policy ${pack.policyId}@${pack.version}: safety ${Math.round(safetyRate * 100)}% < required ${Math.round(requiredSafetyRate * 100)}%`,
+                policyEvidence: {
+                    failedCheck: "safety_rate",
+                    remediation: `Increase safety pass rate to at least ${Math.round(requiredSafetyRate * 100)}%. Review failing test cases for safety-related assertions.`,
+                    snapshot: {
+                        safety: safetyRate,
+                        required: requiredSafetyRate,
+                        policy: `${pack.policyId}@${pack.version}`,
+                    },
+                },
+            };
+        }
+        const violations = policyFlags.filter((f) => maxFlags.includes(f));
+        if (violations.length > 0) {
+            return {
+                exitCode: constants_1.EXIT.POLICY_VIOLATION,
+                passed: false,
+                reasonCode: reason_codes_1.REASON_CODES.POLICY_FAILED,
+                reasonMessage: `policy ${pack.policyId}@${pack.version}: ${violations.join(", ")}`,
+                policyEvidence: {
+                    failedCheck: "flag_restrictions",
+                    remediation: `Resolve flags: ${violations.join(", ")}. These indicate policy violations that must be addressed.`,
+                    snapshot: { violations, policy: `${pack.policyId}@${pack.version}` },
+                },
+            };
+        }
+    }
+    return {
+        exitCode: constants_1.EXIT.PASS,
+        passed: true,
+        reasonCode: reason_codes_1.REASON_CODES.PASS,
+        reasonMessage: null,
+    };
+}

package/dist/cli/impact-analysis.d.ts

@@ -0,0 +1,63 @@
+/**
+ * TICKET 3 — Impact Analysis CLI Command (v0)
+ *
+ * Goal: Modal-like perceived speed via incremental intelligence
+ *
+ * Algorithm v0 (practical, shippable):
+ * - Inputs: manifest.json + git diff --name-only base...HEAD
+ * - Rules: Direct file mapping, dependency tracking, safe fallback
+ * - Output: Human-readable counts + JSON for automation
+ */
+import type { EvaluationManifest } from "./manifest";
+/**
+ * Impact analysis result
+ */
+export interface ImpactAnalysisResult {
+    /** Impacted specification IDs */
+    impactedSpecIds: string[];
+    /** Reason for each impacted spec */
+    reasonBySpecId: Record<string, string>;
+    /** Changed files that triggered the analysis */
+    changedFiles: string[];
+    /** Analysis metadata */
+    metadata: {
+        baseBranch: string;
+        totalSpecs: number;
+        impactedCount: number;
+        analysisTime: number;
+    };
+}
+/**
+ * Impact analysis options
+ */
+export interface ImpactAnalysisOptions {
+    /** Base branch to compare against */
+    baseBranch: string;
+    /** Optional explicit list of changed files (for CI) */
+    changedFiles?: string[];
+    /** Output format */
+    format?: "human" | "json";
+}
+/**
+ * Run impact analysis
+ */
+export declare function runImpactAnalysis(options: ImpactAnalysisOptions, projectRoot?: string): Promise<ImpactAnalysisResult>;
+/**
+ * Analyze impact of changed files
+ */
+export declare function analyzeImpact(changedFiles: string[], manifest: EvaluationManifest): {
+    impactedSpecIds: string[];
+    reasonBySpecId: Record<string, string>;
+};
+/**
+ * Print human-readable results
+ */
+export declare function printHumanResults(result: ImpactAnalysisResult): void;
+/**
+ * Print JSON results
+ */
+export declare function printJsonResults(result: ImpactAnalysisResult): void;
+/**
+ * CLI entry point
+ */
+export declare function runImpactAnalysisCLI(options: ImpactAnalysisOptions): Promise<void>;

package/dist/cli/impact-analysis.js

@@ -0,0 +1,252 @@
+"use strict";
+/**
+ * TICKET 3 — Impact Analysis CLI Command (v0)
+ *
+ * Goal: Modal-like perceived speed via incremental intelligence
+ *
+ * Algorithm v0 (practical, shippable):
+ * - Inputs: manifest.json + git diff --name-only base...HEAD
+ * - Rules: Direct file mapping, dependency tracking, safe fallback
+ * - Output: Human-readable counts + JSON for automation
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runImpactAnalysis = runImpactAnalysis;
+exports.analyzeImpact = analyzeImpact;
+exports.printHumanResults = printHumanResults;
+exports.printJsonResults = printJsonResults;
+exports.runImpactAnalysisCLI = runImpactAnalysisCLI;
+const node_child_process_1 = require("node:child_process");
+const fs = __importStar(require("node:fs/promises"));
+const path = __importStar(require("node:path"));
+/**
+ * Run impact analysis
+ */
+async function runImpactAnalysis(options, projectRoot = process.cwd()) {
+    const startTime = Date.now();
+    // Read manifest
+    const manifest = await readManifest(projectRoot);
+    if (!manifest) {
+        throw new Error("No evaluation manifest found. Run 'evalgate discover --manifest' first.");
+    }
+    // Get changed files
+    const changedFiles = options.changedFiles || (await getChangedFiles(options.baseBranch));
+    // Analyze impact
+    const { impactedSpecIds, reasonBySpecId } = analyzeImpact(changedFiles, manifest);
+    const result = {
+        impactedSpecIds,
+        reasonBySpecId,
+        changedFiles,
+        metadata: {
+            baseBranch: options.baseBranch,
+            totalSpecs: manifest.specs.length,
+            impactedCount: impactedSpecIds.length,
+            analysisTime: Date.now() - startTime,
+        },
+    };
+    return result;
+}
+/**
+ * Read evaluation manifest
+ */
+async function readManifest(projectRoot = process.cwd()) {
+    const manifestPath = path.join(projectRoot, ".evalgate", "manifest.json");
+    try {
+        const content = await fs.readFile(manifestPath, "utf-8");
+        return JSON.parse(content);
+    }
+    catch (_error) {
+        return null;
+    }
+}
+/**
+ * Get changed files from git
+ */
+async function getChangedFiles(baseBranch) {
+    return new Promise((resolve, reject) => {
+        const git = (0, node_child_process_1.spawn)("git", ["diff", "--name-only", `${baseBranch}...HEAD`], {
+            stdio: ["pipe", "pipe", "pipe"],
+        });
+        let output = "";
+        let error = "";
+        git.stdout?.on("data", (data) => {
+            output += data.toString();
+        });
+        git.stderr?.on("data", (data) => {
+            error += data.toString();
+        });
+        git.on("close", (code) => {
+            if (code !== 0) {
+                reject(new Error(`Git diff failed: ${error}`));
+                return;
+            }
+            const files = output
+                .split("\n")
+                .map((f) => f.trim())
+                .filter((f) => f.length > 0)
+                .map((f) => f.replace(/\\/g, "/")); // Normalize to POSIX
+            resolve(files);
+        });
+    });
+}
+/**
+ * Analyze impact of changed files
+ */
+function analyzeImpact(changedFiles, manifest) {
+    const impactedSpecIds = new Set();
+    const reasonBySpecId = {};
+    // Normalize changed files to POSIX format
+    const normalizedChangedFiles = changedFiles.map((f) => f.replace(/\\/g, "/"));
+    // Create lookup maps
+    const specsByFile = new Map();
+    const specsByDependency = new Map();
+    // Index specs by file
+    for (const spec of manifest.specs) {
+        // By file path
+        if (!specsByFile.has(spec.filePath)) {
+            specsByFile.set(spec.filePath, []);
+        }
+        specsByFile.get(spec.filePath)?.push(spec);
+        // By dependencies
+        const deps = [
+            ...spec.dependsOn.prompts,
+            ...spec.dependsOn.datasets,
+            ...spec.dependsOn.tools,
+            ...spec.dependsOn.code,
+        ];
+        for (const dep of deps) {
+            if (!specsByDependency.has(dep)) {
+                specsByDependency.set(dep, []);
+            }
+            specsByDependency.get(dep)?.push(spec);
+        }
+    }
+    // Analyze each changed file
+    for (const changedFile of normalizedChangedFiles) {
+        // Rule 1: Direct spec file change
+        const specsInFile = specsByFile.get(changedFile);
+        if (specsInFile) {
+            for (const spec of specsInFile) {
+                impactedSpecIds.add(spec.id);
+                reasonBySpecId[spec.id] = `Spec file changed: ${changedFile}`;
+            }
+        }
+        // Rule 2: Dependency change
+        const specsUsingDep = specsByDependency.get(changedFile);
+        if (specsUsingDep) {
+            for (const spec of specsUsingDep) {
+                impactedSpecIds.add(spec.id);
+                reasonBySpecId[spec.id] = `Dependency changed: ${changedFile}`;
+            }
+        }
+        // Rule 3: Safe fallback for unknown files
+        if (!specsInFile && !specsUsingDep) {
+            // If we can't map the file, be conservative and run everything
+            console.warn(`⚠️  Unknown changed file: ${changedFile}`);
+            console.warn(`🛡️  Running full suite for safety`);
+            // Add all specs
+            for (const spec of manifest.specs) {
+                impactedSpecIds.add(spec.id);
+                reasonBySpecId[spec.id] =
+                    `Unknown file changed: ${changedFile} (safe fallback)`;
+            }
+            break; // No need to continue analyzing
+        }
+    }
+    return {
+        impactedSpecIds: Array.from(impactedSpecIds).sort(),
+        reasonBySpecId,
+    };
+}
+/**
+ * Print human-readable results
+ */
+function printHumanResults(result) {
+    console.log("\n🔍 Impact Analysis Results");
+    console.log(`📊 Base branch: ${result.metadata.baseBranch}`);
+    console.log(`📁 Changed files: ${result.changedFiles.length}`);
+    console.log(`🎯 Impacted specs: ${result.metadata.impactedCount}/${result.metadata.totalSpecs}`);
+    console.log(`⏱️  Analysis time: ${result.metadata.analysisTime}ms`);
+    if (result.changedFiles.length > 0) {
+        console.log("\n📝 Changed files:");
+        for (const file of result.changedFiles) {
+            console.log(`   • ${file}`);
+        }
+    }
+    if (result.impactedSpecIds.length > 0) {
+        console.log("\n🎯 Impacted specifications:");
+        for (const specId of result.impactedSpecIds) {
+            const reason = result.reasonBySpecId[specId];
+            console.log(`   • ${specId} (${reason})`);
+        }
+        console.log("\n💡 Suggested command:");
+        console.log(`   evalgate run --spec-ids ${result.impactedSpecIds.join(",")}`);
+    }
+    else {
+        console.log("\n✅ No specifications impacted");
+        console.log("💡 No tests needed to run");
+    }
+}
+/**
+ * Print JSON results
+ */
+function printJsonResults(result) {
+    console.log(JSON.stringify(result, null, 2));
+}
+/**
+ * CLI entry point
+ */
+async function runImpactAnalysisCLI(options) {
+    try {
+        const result = await runImpactAnalysis(options);
+        if (options.format === "json") {
+            printJsonResults(result);
+        }
+        else {
+            printHumanResults(result);
+        }
+        // Exit with appropriate code
+        if (result.metadata.impactedCount === 0) {
+            process.exit(0);
+        }
+        else {
+            process.exit(1); // Signal that tests should run
+        }
+    }
+    catch (error) {
+        console.error("❌ Impact analysis failed:", error instanceof Error ? error.message : String(error));
+        process.exit(2);
+    }
+}

package/dist/cli/index.d.ts

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+/**
+ * evalgate — EvalGate CLI
+ *
+ * Commands:
+ *   evalgate init   — Create evalgate.config.json
+ *   evalgate check  — CI/CD evaluation gate (see evalgate check --help)
+ */
+export {};