@evalgate/sdk 2.0.0

package/dist/cli/run.js

@@ -0,0 +1,395 @@
+"use strict";
+/**
+ * TICKET 4 — Unified evalgate run CLI Command
+ *
+ * Goal: Consolidated execution interface that consumes manifest
+ *
+ * Features:
+ * - Manifest loading and spec filtering
+ * - --impacted-only integration with impact analysis
+ * - Local executor integration
+ * - .evalgate/last-run.json output
+ * - Legacy mode compatibility
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runEvaluations = runEvaluations;
+exports.printHumanResults = printHumanResults;
+exports.printJsonResults = printJsonResults;
+exports.runEvaluationsCLI = runEvaluationsCLI;
+const node_child_process_1 = require("node:child_process");
+const fs = __importStar(require("node:fs/promises"));
+const path = __importStar(require("node:path"));
+const impact_analysis_1 = require("./impact-analysis");
+/**
+ * Generate deterministic run ID
+ */
+function generateRunId() {
+    const timestamp = Date.now().toString(36);
+    const random = Math.random().toString(36).substring(2, 8);
+    return `run-${timestamp}-${random}`;
+}
+/**
+ * Run evaluation specifications
+ */
+async function runEvaluations(options, projectRoot = process.cwd()) {
+    const startTime = Date.now();
+    // Load manifest
+    const manifest = await loadManifest(projectRoot);
+    if (!manifest) {
+        throw new Error("No evaluation manifest found. Run 'evalgate discover --manifest' first.");
+    }
+    // Determine which specs to run
+    let specsToRun = manifest.specs;
+    if (options.impactedOnly && options.baseBranch) {
+        // Run impact analysis first
+        const impactResult = await (0, impact_analysis_1.runImpactAnalysis)({
+            baseBranch: options.baseBranch,
+        }, projectRoot);
+        // Filter to impacted specs only
+        const impactedSpecIds = new Set(impactResult.impactedSpecIds);
+        specsToRun = manifest.specs.filter((spec) => impactedSpecIds.has(spec.id));
+        console.log(`🎯 Running ${specsToRun.length} impacted specs (out of ${manifest.specs.length} total)`);
+    }
+    else if (options.specIds && options.specIds.length > 0) {
+        // Filter to specific spec IDs
+        const specIdSet = new Set(options.specIds);
+        specsToRun = manifest.specs.filter((spec) => specIdSet.has(spec.id));
+        console.log(`🎯 Running ${specsToRun.length} specific specs`);
+    }
+    else if (options.specIds && options.specIds.length === 0) {
+        // Explicit empty list means run nothing
+        specsToRun = [];
+        console.log(`🎯 Running 0 specs (explicit empty list)`);
+    }
+    else {
+        console.log(`🎯 Running all ${specsToRun.length} specs`);
+    }
+    // Execute specs
+    const results = await executeSpecs(specsToRun);
+    const completedAt = Date.now();
+    const duration = completedAt - startTime;
+    // Calculate summary
+    const summary = calculateSummary(results);
+    const runResult = {
+        schemaVersion: 1,
+        runId: generateRunId(),
+        metadata: {
+            startedAt: startTime,
+            completedAt,
+            duration,
+            totalSpecs: manifest.specs.length,
+            executedSpecs: specsToRun.length,
+            mode: manifest.runtime.mode,
+        },
+        results,
+        summary,
+    };
+    // Write results if requested
+    if (options.writeResults) {
+        await writeRunResults(runResult, projectRoot);
+        await updateRunIndex(runResult, projectRoot);
+    }
+    return runResult;
+}
+/**
+ * Load evaluation manifest
+ */
+async function loadManifest(projectRoot = process.cwd()) {
+    const manifestPath = path.join(projectRoot, ".evalgate", "manifest.json");
+    try {
+        const content = await fs.readFile(manifestPath, "utf-8");
+        return JSON.parse(content);
+    }
+    catch (_error) {
+        return null;
+    }
+}
+/**
+ * Execute specifications
+ */
+async function executeSpecs(specs) {
+    const results = [];
+    for (const spec of specs) {
+        const result = await executeSpec(spec);
+        results.push(result);
+    }
+    return results;
+}
+/**
+ * Execute individual specification
+ */
+async function executeSpec(spec) {
+    const startTime = Date.now();
+    try {
+        // For now, simulate execution
+        // In a real implementation, this would:
+        // 1. Load the spec file
+        // 2. Execute the defineEval function
+        // 3. Capture the result
+        // Simulate some work
+        await new Promise((resolve) => setTimeout(resolve, Math.random() * 100 + 50));
+        // Simulate success/failure (90% success rate for demo)
+        const success = Math.random() > 0.1;
+        const duration = Date.now() - startTime;
+        if (success) {
+            return {
+                specId: spec.id,
+                name: spec.name,
+                filePath: spec.filePath,
+                result: {
+                    status: "passed",
+                    score: Math.random() * 0.3 + 0.7, // 0.7-1.0
+                    duration,
+                },
+            };
+        }
+        else {
+            return {
+                specId: spec.id,
+                name: spec.name,
+                filePath: spec.filePath,
+                result: {
+                    status: "failed",
+                    error: "Simulated execution failure",
+                    duration,
+                },
+            };
+        }
+    }
+    catch (error) {
+        return {
+            specId: spec.id,
+            name: spec.name,
+            filePath: spec.filePath,
+            result: {
+                status: "failed",
+                error: error instanceof Error ? error.message : String(error),
+                duration: Date.now() - startTime,
+            },
+        };
+    }
+}
+/**
+ * Calculate summary statistics
+ */
+function calculateSummary(results) {
+    const passed = results.filter((r) => r.result.status === "passed").length;
+    const failed = results.filter((r) => r.result.status === "failed").length;
+    const skipped = results.filter((r) => r.result.status === "skipped").length;
+    const passRate = results.length > 0 ? passed / results.length : 0;
+    return {
+        passed,
+        failed,
+        skipped,
+        passRate,
+    };
+}
+/**
+ * Write run results to file
+ */
+async function writeRunResults(result, projectRoot = process.cwd()) {
+    const evalgateDir = path.join(projectRoot, ".evalgate");
+    await fs.mkdir(evalgateDir, { recursive: true });
+    // Write last-run.json (existing behavior)
+    const lastRunPath = path.join(evalgateDir, "last-run.json");
+    await fs.writeFile(lastRunPath, JSON.stringify(result, null, 2), "utf-8");
+    // Create runs directory and write timestamped artifact
+    if (result.runId) {
+        const runsDir = path.join(evalgateDir, "runs");
+        await fs.mkdir(runsDir, { recursive: true });
+        const timestampedPath = path.join(runsDir, `${result.runId}.json`);
+        await fs.writeFile(timestampedPath, JSON.stringify(result, null, 2), "utf-8");
+        // Optional: Create latest.json mirror
+        const latestPath = path.join(runsDir, "latest.json");
+        await fs.writeFile(latestPath, JSON.stringify(result, null, 2), "utf-8");
+    }
+    console.log(`✅ Run results written to .evalgate/last-run.json`);
+    if (result.runId) {
+        console.log(`📁 Run artifact: .evalgate/runs/${result.runId}.json`);
+    }
+}
+/**
+ * Update run index with new run entry
+ */
+async function updateRunIndex(result, projectRoot = process.cwd()) {
+    const runsDir = path.join(projectRoot, ".evalgate", "runs");
+    const indexPath = path.join(runsDir, "index.json");
+    await fs.mkdir(runsDir, { recursive: true });
+    // Calculate average score
+    const scores = result.results
+        .filter((r) => r.result.score !== undefined)
+        .map((r) => r.result.score);
+    const avgScore = scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0;
+    // Get git info if available
+    let gitSha;
+    let branch;
+    try {
+        gitSha = await getGitSha();
+        branch = await getGitBranch();
+    }
+    catch {
+        // Git commands not available, continue without git info
+    }
+    const indexEntry = {
+        runId: result.runId,
+        createdAt: result.metadata.startedAt,
+        gitSha,
+        branch,
+        mode: result.metadata.mode,
+        specCount: result.results.length,
+        passRate: result.summary.passRate,
+        avgScore,
+    };
+    // Read existing index or create new one
+    let index = [];
+    try {
+        const existingContent = await fs.readFile(indexPath, "utf-8");
+        index = JSON.parse(existingContent);
+    }
+    catch (_error) {
+        // Index doesn't exist yet, start with empty array
+    }
+    // Add new entry
+    index.push(indexEntry);
+    // Sort by creation time (newest first)
+    index.sort((a, b) => b.createdAt - a.createdAt);
+    // Write to temp file first, then rename for atomicity
+    const tempPath = `${indexPath}.tmp`;
+    await fs.writeFile(tempPath, JSON.stringify(index, null, 2), "utf-8");
+    await fs.rename(tempPath, indexPath);
+}
+/**
+ * Get current git SHA
+ */
+async function getGitSha() {
+    return new Promise((resolve) => {
+        const git = (0, node_child_process_1.spawn)("git", ["rev-parse", "HEAD"], {
+            stdio: ["pipe", "pipe", "pipe"],
+        });
+        let output = "";
+        git.stdout.on("data", (data) => {
+            output += data.toString();
+        });
+        git.on("close", (code) => {
+            if (code === 0 && output.trim()) {
+                resolve(output.trim());
+            }
+            else {
+                resolve(undefined);
+            }
+        });
+    });
+}
+/**
+ * Get current git branch
+ */
+async function getGitBranch() {
+    return new Promise((resolve) => {
+        const git = (0, node_child_process_1.spawn)("git", ["rev-parse", "--abbrev-ref", "HEAD"], {
+            stdio: ["pipe", "pipe", "pipe"],
+        });
+        let output = "";
+        git.stdout.on("data", (data) => {
+            output += data.toString();
+        });
+        git.on("close", (code) => {
+            if (code === 0 && output.trim()) {
+                resolve(output.trim());
+            }
+            else {
+                resolve(undefined);
+            }
+        });
+    });
+}
+/**
+ * Print human-readable results
+ */
+function printHumanResults(result) {
+    console.log("\n🏃 Evaluation Run Results");
+    console.log(`⏱️  Duration: ${result.metadata.duration}ms`);
+    console.log(`📊 Specs: ${result.metadata.executedSpecs}/${result.metadata.totalSpecs} executed`);
+    console.log(`🎯 Mode: ${result.metadata.mode}`);
+    console.log("\n📈 Summary:");
+    console.log(`   ✅ Passed: ${result.summary.passed}`);
+    console.log(`   ❌ Failed: ${result.summary.failed}`);
+    console.log(`   ⏭️  Skipped: ${result.summary.skipped}`);
+    console.log(`   📊 Pass Rate: ${(result.summary.passRate * 100).toFixed(1)}%`);
+    console.log("\n📋 Individual Results:");
+    for (const spec of result.results) {
+        const status = spec.result.status === "passed"
+            ? "✅"
+            : spec.result.status === "failed"
+                ? "❌"
+                : "⏭️";
+        const score = spec.result.score
+            ? ` (${(spec.result.score * 100).toFixed(1)}%)`
+            : "";
+        const error = spec.result.error ? ` - ${spec.result.error}` : "";
+        console.log(`   ${status} ${spec.name}${score}${error}`);
+    }
+}
+/**
+ * Print JSON results
+ */
+function printJsonResults(result) {
+    console.log(JSON.stringify(result, null, 2));
+}
+/**
+ * CLI entry point
+ */
+async function runEvaluationsCLI(options) {
+    try {
+        const result = await runEvaluations(options);
+        if (options.format === "json") {
+            printJsonResults(result);
+        }
+        else {
+            printHumanResults(result);
+        }
+        // Exit with appropriate code
+        if (result.summary.failed > 0) {
+            process.exit(1);
+        }
+        else {
+            process.exit(0);
+        }
+    }
+    catch (error) {
+        console.error("❌ Run failed:", error instanceof Error ? error.message : String(error));
+        process.exit(2);
+    }
+}

package/dist/cli/share.d.ts

@@ -0,0 +1,17 @@
+/**
+ * evalgate share — Create a share link for a run.
+ * Usage: evalgate share --scope run --expires 7d
+ */
+export type ShareArgs = {
+    baseUrl: string;
+    apiKey: string;
+    evaluationId: string;
+    runId: number;
+    scope: "run";
+    expires: string;
+    expiresInDays: number;
+};
+export declare function parseShareArgs(argv: string[]): ShareArgs | {
+    error: string;
+};
+export declare function runShare(args: ShareArgs): Promise<number>;

package/dist/cli/share.js

@@ -0,0 +1,91 @@
+"use strict";
+/**
+ * evalgate share — Create a share link for a run.
+ * Usage: evalgate share --scope run --expires 7d
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseShareArgs = parseShareArgs;
+exports.runShare = runShare;
+const api_1 = require("./api");
+function parseExpires(spec) {
+    const m = spec.match(/^(\d+)(d|h|m|s)$/i);
+    if (!m)
+        return null;
+    const n = parseInt(m[1], 10);
+    const unit = m[2].toLowerCase();
+    if (unit === "d")
+        return n;
+    if (unit === "h")
+        return n / 24;
+    if (unit === "m")
+        return n / (24 * 60);
+    if (unit === "s")
+        return n / (24 * 60 * 60);
+    return null;
+}
+function parseShareArgs(argv) {
+    const args = {};
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        if (arg.startsWith("--")) {
+            const key = arg.slice(2);
+            const next = argv[i + 1];
+            if (next !== undefined && !next.startsWith("--")) {
+                args[key] = next;
+                i++;
+            }
+            else {
+                args[key] = "true";
+            }
+        }
+    }
+    const baseUrl = args.baseUrl ||
+        process.env.EVALGATE_BASE_URL ||
+        process.env.EVALAI_BASE_URL ||
+        "http://localhost:3000";
+    const apiKey = args.apiKey ||
+        process.env.EVALGATE_API_KEY ||
+        process.env.EVALAI_API_KEY ||
+        "";
+    const evaluationId = args.evaluationId || "";
+    const runId = args.runId ? parseInt(args.runId, 10) : NaN;
+    const scope = args.scope === "run" ? "run" : "run";
+    const expires = args.expires || "7d";
+    if (!apiKey)
+        return { error: "Error: --apiKey or EVALGATE_API_KEY is required" };
+    if (!evaluationId)
+        return { error: "Error: --evaluationId is required" };
+    if (Number.isNaN(runId) || runId < 1)
+        return {
+            error: "Error: --runId is required and must be a positive number",
+        };
+    const expiresInDays = parseExpires(expires);
+    if (expiresInDays == null || expiresInDays <= 0)
+        return { error: "Error: --expires must be e.g. 7d, 24h, 60m, 1s" };
+    return {
+        baseUrl,
+        apiKey,
+        evaluationId,
+        runId,
+        scope,
+        expires,
+        expiresInDays,
+    };
+}
+async function runShare(args) {
+    const exportRes = await (0, api_1.fetchRunExport)(args.baseUrl, args.apiKey, args.evaluationId, args.runId);
+    if (!exportRes.ok) {
+        console.error(`EvalGate share: failed to fetch export — ${exportRes.status} ${exportRes.body}`);
+        return 1;
+    }
+    const publishRes = await (0, api_1.publishShare)(args.baseUrl, args.apiKey, args.evaluationId, exportRes.exportData, args.runId, { expiresInDays: args.expiresInDays });
+    if (!publishRes.ok) {
+        console.error(`EvalGate share: failed to publish — ${publishRes.status} ${publishRes.body}`);
+        return 1;
+    }
+    const shareUrl = publishRes.data.shareUrl ??
+        `${args.baseUrl.replace(/\/$/, "")}/share/${publishRes.data.shareId}`;
+    console.log(`Share link created (expires in ${args.expires}):`);
+    console.log(shareUrl);
+    return 0;
+}

package/dist/cli/upgrade.d.ts

@@ -0,0 +1,15 @@
+/**
+ * evalgate upgrade --full — Upgrade from Tier 1 (built-in gate) to Tier 2 (full gate)
+ *
+ * What it does:
+ *   1. Adds full regression gate script (scripts/regression-gate.ts)
+ *   2. Adds baseline governance workflow (.github/workflows/baseline-governance.yml)
+ *   3. Updates package.json with eval:regression-gate + eval:baseline-update scripts
+ *   4. Updates .github/workflows/evalgate-gate.yml to use project mode
+ *   5. Prints next steps
+ */
+export interface UpgradeArgs {
+    full: boolean;
+}
+export declare function parseUpgradeArgs(argv: string[]): UpgradeArgs;
+export declare function runUpgrade(argv: string[]): number;