@pauly4010/evalai-sdk 1.8.0 → 1.9.0

package/dist/cli/diff.js

@@ -0,0 +1,680 @@
+"use strict";
+/**
+ * TICKET 5 — Behavioral Diff CLI (EVAL-401)
+ *
+ * Goal: "Git diff for AI behavior" from two RunReports
+ *
+ * Command:
+ * evalai diff --base main (default uses git to find baseline run)
+ * evalai diff --a <runReportPath> --b <runReportPath>
+ * evalai diff main..feature (nice-to-have alias)
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.diffCore = exports.SUPPORTED_SCHEMA_VERSIONS = exports.DIFF_SCHEMA_VERSION = void 0;
+exports.round = round;
+exports.roundPct = roundPct;
+exports.validateSchemaVersion = validateSchemaVersion;
+exports.runDiff = runDiff;
+exports.compareReports = compareReports;
+exports.calculateDiffSummary = calculateDiffSummary;
+exports.printHumanResults = printHumanResults;
+exports.printJsonResults = printJsonResults;
+exports.writeGitHubStepSummary = writeGitHubStepSummary;
+exports.runDiffCLI = runDiffCLI;
+exports.classifyDiff = classifyDiff;
+exports.calculateDeltas = calculateDeltas;
+const node_child_process_1 = require("node:child_process");
+const fs = __importStar(require("node:fs/promises"));
+const path = __importStar(require("node:path"));
+const env_1 = require("./env");
+const workspace_1 = require("./workspace");
+/**
+ * Diff schema version
+ */
+exports.DIFF_SCHEMA_VERSION = 1;
+/**
+ * Supported RunReport schema versions
+ */
+exports.SUPPORTED_SCHEMA_VERSIONS = [1];
+/**
+ * Rounding helpers for floating point normalization
+ */
+function round(value, precision = 4) {
+    return Math.round(value * 10 ** precision) / 10 ** precision;
+}
+function roundPct(value, precision = 1) {
+    return round(value * 100, precision);
+}
+/**
+ * Validate RunReport schema version
+ */
+function validateSchemaVersion(report) {
+    if (!report.schemaVersion) {
+        throw new Error(`RunReport missing schemaVersion. This report was generated by an older version of EvalAI.\n` +
+            `Please regenerate the run report or upgrade to a compatible version.`);
+    }
+    if (!exports.SUPPORTED_SCHEMA_VERSIONS.includes(report.schemaVersion)) {
+        throw new Error(`Unsupported RunReport schema version: ${report.schemaVersion}\n` +
+            `Supported versions: ${exports.SUPPORTED_SCHEMA_VERSIONS.join(", ")}\n` +
+            `This report was generated by a newer version of EvalAI. Please upgrade your EvalAI CLI.`);
+    }
+}
+/**
+ * Check if running in CI environment
+ */
+function isCIEnvironment() {
+    return (0, env_1.isCI)();
+}
+/**
+ * Generate CI-friendly error message for missing base
+ */
+function generateCIBaseErrorMessage(baseRef) {
+    const lines = [
+        "🚫 Base run report not found in CI environment",
+        "",
+        "To fix this, download the base run artifact from your base branch workflow:",
+        "",
+        `1. Download run artifact from ${baseRef} branch workflow`,
+        "2. Save it as .evalai/base-run.json",
+        "3. Re-run: evalai diff --base .evalai/base-run.json --head .evalai/last-run.json",
+        "",
+        "Expected artifact patterns:",
+        "  - .evalai/runs/run-*.json",
+        "  - .evalai/last-run.json",
+        "",
+        "Or set a baseline pointer:",
+        "  evalai diff --base <runId> --head last --set-baseline",
+    ];
+    return lines.join("\n");
+}
+/**
+ * Run diff comparison
+ */
+async function runDiff(options) {
+    // Resolve base and head reports
+    const baseReport = await resolveBaseReport(options.base);
+    const headReport = await resolveHeadReport(options.head);
+    if (!baseReport) {
+        if (isCIEnvironment() && options.base && (0, env_1.isGitRef)(options.base)) {
+            // In CI with git ref, provide helpful guidance
+            throw new Error(generateCIBaseErrorMessage(options.base));
+        }
+        throw new Error("Base run report not found. Use --base to specify a report or branch.");
+    }
+    if (!headReport) {
+        throw new Error("Head run report not found. Use --head to specify a report path.");
+    }
+    // Validate schema versions
+    validateSchemaVersion(baseReport);
+    validateSchemaVersion(headReport);
+    // Perform diff comparison
+    const diffResult = compareReports(baseReport, headReport);
+    return diffResult;
+}
+/**
+ * Resolve base report from options
+ */
+async function resolveBaseReport(base) {
+    if (!base) {
+        // Default: try to find last run for main branch
+        return await findLastRunForBranch("main");
+    }
+    if (base === "last") {
+        // Resolve to previous run in index
+        return await findPreviousRun();
+    }
+    if (base === "baseline") {
+        // Resolve to baseline pointer or runId in index
+        return await findBaselineRun();
+    }
+    if (base.includes("..")) {
+        // Branch range like "main..feature" - extract base branch
+        const baseBranch = base.split("..")[0];
+        return await findLastRunForBranch(baseBranch);
+    }
+    // Check if it's a branch name
+    if (await isBranchName(base)) {
+        return await findLastRunForBranch(base);
+    }
+    // Check if it's a runId
+    if (base.startsWith("run-")) {
+        return await loadRunReport(`.evalai/runs/${base}.json`);
+    }
+    // Treat as file path
+    return await loadRunReport(base);
+}
+/**
+ * Find baseline run from pointer file or index
+ */
+async function findBaselineRun() {
+    const workspace = (0, workspace_1.resolveEvalWorkspace)();
+    // First, check for baseline pointer file
+    try {
+        const content = await fs.readFile(workspace.baselinePath, "utf-8");
+        return JSON.parse(content);
+    }
+    catch (error) {
+        // Baseline file doesn't exist, try index
+    }
+    // Check index for baseline runId
+    try {
+        const content = await fs.readFile(workspace.indexPath, "utf-8");
+        const index = JSON.parse(content);
+        // Look for a run marked as baseline (could be a future enhancement)
+        // For now, just return the oldest run in index
+        if (index.length > 0) {
+            const oldestRunId = index[index.length - 1].runId;
+            return await loadRunReport(`.evalai/runs/${oldestRunId}.json`);
+        }
+    }
+    catch (error) {
+        // Index doesn't exist
+    }
+    throw new Error("No baseline run found. Set a baseline with 'evalai diff --base <runId> --head last --set-baseline' or create .evalai/baseline-run.json.");
+}
+/**
+ * Resolve head report from options
+ */
+async function resolveHeadReport(head) {
+    if (head) {
+        if (head === "last") {
+            return await loadRunReport(".evalai/last-run.json");
+        }
+        // Check if it's a runId
+        if (head.startsWith("run-")) {
+            return await loadRunReport(`.evalai/runs/${head}.json`);
+        }
+        // Treat as file path
+        return await loadRunReport(head);
+    }
+    // Default: use last run
+    return await loadRunReport(".evalai/last-run.json");
+}
+/**
+ * Find previous run from index
+ */
+async function findPreviousRun() {
+    const indexPath = path.join(process.cwd(), ".evalai", "runs", "index.json");
+    try {
+        const content = await fs.readFile(indexPath, "utf-8");
+        const index = JSON.parse(content);
+        if (index.length < 2) {
+            throw new Error("Need at least 2 runs to use 'last' shortcut. Run 'evalai run --write-results' at least twice.");
+        }
+        // Return the second most recent run (index[0] is the most recent)
+        const previousRunId = index[1].runId;
+        return await loadRunReport(`.evalai/runs/${previousRunId}.json`);
+    }
+    catch (error) {
+        if (error instanceof Error && error.message.includes("Need at least 2 runs")) {
+            throw error;
+        }
+        throw new Error("No run history found. Run 'evalai run --write-results' first.");
+    }
+}
+/**
+ * Check if string is a branch name
+ */
+async function isBranchName(name) {
+    return new Promise((resolve) => {
+        const git = (0, node_child_process_1.spawn)("git", ["rev-parse", "--verify", name], {
+            stdio: ["pipe", "pipe", "pipe"],
+        });
+        git.on("close", (code) => {
+            resolve(code === 0);
+        });
+    });
+}
+/**
+ * Find last run for a branch
+ */
+async function findLastRunForBranch(branch) {
+    // For now, just look for .evalai/last-run.json
+    // In a real implementation, this would:
+    // 1. Check CI artifacts for the branch
+    // 2. Look for branch-specific run files
+    // 3. Fall back to local last-run.json
+    const lastRunPath = path.join(process.cwd(), ".evalai", "last-run.json");
+    try {
+        const content = await fs.readFile(lastRunPath, "utf-8");
+        return JSON.parse(content);
+    }
+    catch (error) {
+        return null;
+    }
+}
+/**
+ * Load run report from file
+ */
+async function loadRunReport(filePath) {
+    try {
+        const content = await fs.readFile(path.resolve(filePath), "utf-8");
+        return JSON.parse(content);
+    }
+    catch (error) {
+        return null;
+    }
+}
+/**
+ * Compare two run reports
+ */
+function compareReports(base, head) {
+    const changedSpecs = [];
+    // Create maps for easy lookup
+    const baseSpecs = new Map(base.results.map((r) => [r.specId, r]));
+    const headSpecs = new Map(head.results.map((r) => [r.specId, r]));
+    const allSpecIds = new Set([...baseSpecs.keys(), ...headSpecs.keys()]);
+    // Analyze each spec
+    for (const specId of allSpecIds) {
+        const baseSpec = baseSpecs.get(specId);
+        const headSpec = headSpecs.get(specId);
+        const specDiff = analyzeSpecDiff(specId, baseSpec, headSpec);
+        if (specDiff) {
+            changedSpecs.push(specDiff);
+        }
+    }
+    // Sort by severity then ID
+    changedSpecs.sort((a, b) => {
+        const severityOrder = getSeverityOrder(a.classification) - getSeverityOrder(b.classification);
+        if (severityOrder !== 0)
+            return severityOrder;
+        return a.specId.localeCompare(b.specId);
+    });
+    // Calculate summary
+    const summary = calculateDiffSummary(base, head, changedSpecs);
+    return {
+        schemaVersion: exports.DIFF_SCHEMA_VERSION,
+        base,
+        head,
+        summary,
+        changedSpecs,
+        metadata: {
+            generatedAt: Date.now(),
+            baseSource: "local",
+            headSource: "local",
+        },
+    };
+}
+/**
+ * Analyze diff for a single spec
+ */
+function analyzeSpecDiff(specId, base, head) {
+    const name = head?.name || base?.name || specId;
+    const filePath = head?.filePath || base?.filePath || "";
+    // Determine classification
+    const classification = classifyDiff(base, head);
+    // If no significant change, return null
+    if (classification === "execution_error" && base && head) {
+        // Check if it's actually identical
+        if (base.result.status === head.result.status &&
+            (base.result.score || 0) === (head.result.score || 0) &&
+            base.result.duration === head.result.duration &&
+            base.result.error === head.result.error) {
+            return null;
+        }
+    }
+    // Calculate deltas
+    const deltas = calculateDeltas(base, head);
+    return {
+        specId,
+        name,
+        filePath,
+        classification,
+        base: base
+            ? {
+                status: base.result.status,
+                score: base.result.score,
+                duration: base.result.duration,
+                error: base.result.error,
+            }
+            : undefined,
+        head: head
+            ? {
+                status: head.result.status,
+                score: head.result.score,
+                duration: head.result.duration,
+                error: head.result.error,
+            }
+            : undefined,
+        deltas,
+    };
+}
+/**
+ * Classify the type of change
+ */
+function classifyDiff(base, head) {
+    if (!base && head) {
+        return "added";
+    }
+    if (base && !head) {
+        return "removed";
+    }
+    if (!base || !head) {
+        return "execution_error";
+    }
+    // Both exist - analyze changes
+    if (base.result.status === "passed" && head.result.status === "failed") {
+        return "new_failure";
+    }
+    if (base.result.status === "failed" && head.result.status === "passed") {
+        return "fixed_failure";
+    }
+    if (base.result.status === "skipped" && head.result.status !== "skipped") {
+        return "skipped_change";
+    }
+    if (head.result.status === "skipped" && base.result.status !== "skipped") {
+        return "skipped_change";
+    }
+    // Score changes
+    if (base.result.score && head.result.score) {
+        const delta = head.result.score - base.result.score;
+        if (delta < -0.05)
+            return "score_drop";
+        if (delta > 0.05)
+            return "score_improve";
+    }
+    // Default to no significant change
+    return "execution_error";
+}
+/**
+ * Calculate deltas between base and head
+ */
+function calculateDeltas(base, head) {
+    const deltas = {};
+    if (base && head) {
+        if (base.result.score && head.result.score) {
+            deltas.scoreDelta = round(head.result.score - base.result.score, 4);
+        }
+        deltas.durationDelta = head.result.duration - base.result.duration;
+        if (base.result.status !== head.result.status) {
+            deltas.statusChange = `${base.result.status} → ${head.result.status}`;
+        }
+    }
+    return deltas;
+}
+/**
+ * Get severity order for sorting
+ */
+function getSeverityOrder(classification) {
+    const severityMap = {
+        new_failure: 1,
+        score_drop: 2,
+        execution_error: 3,
+        skipped_change: 4,
+        removed: 5,
+        added: 6,
+        fixed_failure: 7,
+        score_improve: 8,
+    };
+    return severityMap[classification] || 9;
+}
+/**
+ * Calculate diff summary statistics
+ */
+function calculateDiffSummary(base, head, changedSpecs) {
+    const baseTotal = base.results.length;
+    const headTotal = head.results.length;
+    const basePassRate = base.summary.passRate;
+    const headPassRate = head.summary.passRate;
+    const passRateDelta = round(headPassRate - basePassRate, 4);
+    // Calculate average scores
+    const baseScores = base.results
+        .filter((r) => r.result.score !== undefined)
+        .map((r) => r.result.score);
+    const headScores = head.results
+        .filter((r) => r.result.score !== undefined)
+        .map((r) => r.result.score);
+    const baseAvgScore = baseScores.length > 0 ? baseScores.reduce((a, b) => a + b, 0) / baseScores.length : 0;
+    const headAvgScore = headScores.length > 0 ? headScores.reduce((a, b) => a + b, 0) / headScores.length : 0;
+    const scoreDelta = round(headAvgScore - baseAvgScore, 4);
+    // Count classifications
+    const regressions = changedSpecs.filter((s) => ["new_failure", "score_drop", "execution_error"].includes(s.classification)).length;
+    const improvements = changedSpecs.filter((s) => ["fixed_failure", "score_improve"].includes(s.classification)).length;
+    const added = changedSpecs.filter((s) => s.classification === "added").length;
+    const removed = changedSpecs.filter((s) => s.classification === "removed").length;
+    return {
+        baseTotal,
+        headTotal,
+        passRateDelta,
+        scoreDelta,
+        regressions,
+        improvements,
+        added,
+        removed,
+    };
+}
+/**
+ * Print human-readable diff results
+ */
+function printHumanResults(result) {
+    console.log("\n🔄 Behavioral Diff Results");
+    console.log(`📊 Base: ${result.metadata.baseSource} (${result.summary.baseTotal} specs)`);
+    console.log(`📈 Head: ${result.metadata.headSource} (${result.summary.headTotal} specs)`);
+    console.log("\n📈 Summary:");
+    console.log(`   📊 Pass Rate Delta: ${roundPct(result.summary.passRateDelta, 1).toFixed(1)}%`);
+    console.log(`   🎯 Score Delta: ${roundPct(result.summary.scoreDelta, 1).toFixed(1)}%`);
+    console.log(`   📉 Regressions: ${result.summary.regressions}`);
+    console.log(`   📈 Improvements: ${result.summary.improvements}`);
+    console.log(`   ➕ Added: ${result.summary.added}`);
+    console.log(`   ➖ Removed: ${result.summary.removed}`);
+    if (result.changedSpecs.length > 0) {
+        console.log("\n🔍 Changed Specifications:");
+        for (const spec of result.changedSpecs) {
+            const icon = getClassificationIcon(spec.classification);
+            const scoreInfo = spec.deltas.scoreDelta
+                ? ` (${spec.deltas.scoreDelta > 0 ? "+" : ""}${roundPct(spec.deltas.scoreDelta, 1).toFixed(1)}%)`
+                : "";
+            const durationInfo = spec.deltas.durationDelta
+                ? ` (${spec.deltas.durationDelta > 0 ? "+" : ""}${spec.deltas.durationDelta}ms)`
+                : "";
+            console.log(`   ${icon} ${spec.name}${scoreInfo}${durationInfo}`);
+            if (spec.head?.error) {
+                console.log(`      ❌ ${spec.head.error}`);
+            }
+        }
+    }
+    else {
+        console.log("\n✅ No changes detected");
+    }
+}
+/**
+ * Get icon for classification
+ */
+function getClassificationIcon(classification) {
+    const iconMap = {
+        new_failure: "🆘",
+        fixed_failure: "✅",
+        score_drop: "📉",
+        score_improve: "📈",
+        execution_error: "❌",
+        skipped_change: "⏭️",
+        added: "➕",
+        removed: "➖",
+    };
+    return iconMap[classification] || "❓";
+}
+/**
+ * Print JSON results
+ */
+function printJsonResults(result) {
+    console.log(JSON.stringify(result, null, 2));
+}
+/**
+ * Write GitHub Step Summary
+ */
+async function writeGitHubStepSummary(result) {
+    if (!process.env.GITHUB_STEP_SUMMARY) {
+        return; // Not in GitHub Actions
+    }
+    const summaryPath = process.env.GITHUB_STEP_SUMMARY;
+    try {
+        const summary = generateGitHubSummary(result);
+        await fs.appendFile(summaryPath, summary + "\n", "utf-8");
+    }
+    catch (error) {
+        console.warn("Warning: Could not write GitHub Step Summary:", error);
+    }
+}
+/**
+ * Generate GitHub Step Summary content
+ */
+function generateGitHubSummary(result) {
+    const lines = [];
+    // Header
+    lines.push("## 🤖 EvalAI Diff Results\n");
+    // Summary metrics
+    lines.push("### 📊 Summary Metrics");
+    lines.push(`- **Pass Rate Delta**: ${roundPct(result.summary.passRateDelta, 1).toFixed(1)}%`);
+    lines.push(`- **Score Delta**: ${roundPct(result.summary.scoreDelta, 1).toFixed(1)}%`);
+    lines.push(`- **🚨 Regressions**: ${result.summary.regressions}`);
+    lines.push(`- **📈 Improvements**: ${result.summary.improvements}`);
+    lines.push(`- **➕ Added**: ${result.summary.added}`);
+    lines.push(`- **➖ Removed**: ${result.summary.removed}`);
+    lines.push("");
+    // Status indicator
+    if (result.summary.regressions > 0) {
+        lines.push("### 🚨 Regressions Detected\n");
+        lines.push("**⚠️ This PR contains regressions that should be reviewed.**\n");
+    }
+    else {
+        lines.push("### ✅ No Regressions Detected\n");
+        lines.push("**🎉 All tests passed! No regressions found.**\n");
+    }
+    // Top regressions
+    const regressions = result.changedSpecs.filter((s) => ["new_failure", "execution_error", "score_drop"].includes(s.classification));
+    if (regressions.length > 0) {
+        lines.push("### 🔍 Top Regressions\n");
+        const topRegressions = regressions.slice(0, 5);
+        for (const spec of topRegressions) {
+            const icon = getClassificationIcon(spec.classification);
+            const scoreInfo = spec.deltas.scoreDelta
+                ? ` (${spec.deltas.scoreDelta > 0 ? "+" : ""}${roundPct(spec.deltas.scoreDelta, 1).toFixed(1)}%)`
+                : "";
+            lines.push(`${icon} **${spec.name}**${scoreInfo}`);
+            lines.push(`   Classification: \`${spec.classification}\``);
+            if (spec.head?.error) {
+                lines.push(`   Error: \`${spec.head.error}\``);
+            }
+            lines.push("");
+        }
+        if (regressions.length > 5) {
+            lines.push(`... and ${regressions.length - 5} more regressions\n`);
+        }
+    }
+    // Artifacts
+    lines.push("### 📁 Artifacts\n");
+    lines.push(`- **Run Index**: \`.evalai/runs/index.json\``);
+    lines.push(`- **Latest Run**: \`.evalai/runs/${result.head.runId}.json\``);
+    lines.push(`- **Last Run**: \`.evalai/last-run.json\``);
+    lines.push("");
+    // Footer
+    lines.push("<details>");
+    lines.push("<summary>🔧 Technical Details</summary>");
+    lines.push("");
+    lines.push("```json");
+    lines.push(JSON.stringify({
+        baseRunId: result.base.runId,
+        headRunId: result.head.runId,
+        baseTotal: result.summary.baseTotal,
+        headTotal: result.summary.headTotal,
+        passRateDelta: result.summary.passRateDelta,
+        scoreDelta: result.summary.scoreDelta,
+        regressions: result.summary.regressions,
+        improvements: result.summary.improvements,
+        added: result.summary.added,
+        removed: result.summary.removed,
+    }, null, 2));
+    lines.push("```");
+    lines.push("");
+    lines.push("</details>");
+    return lines.join("\n");
+}
+/**
+ * CLI entry point
+ */
+async function runDiffCLI(options) {
+    try {
+        const result = await runDiff(options);
+        if (options.format === "json") {
+            printJsonResults(result);
+        }
+        else {
+            printHumanResults(result);
+        }
+        // Write GitHub Step Summary if in CI
+        await writeGitHubStepSummary(result);
+        // Exit with appropriate code
+        if (result.summary.regressions > 0) {
+            process.exit(1); // Regressions detected
+        }
+        else {
+            process.exit(0); // Clean
+        }
+    }
+    catch (error) {
+        console.error(`EvalAI ERROR: ${error instanceof Error ? error.message : String(error)}`);
+        // In CI with git ref error, exit with config code
+        if (isCIEnvironment() &&
+            options.base &&
+            (0, env_1.isGitRef)(options.base) &&
+            error instanceof Error &&
+            error.message.includes("Base run report not found in CI environment")) {
+            process.exit(2); // Config/infra issue
+        }
+        else {
+            process.exit(2); // General error
+        }
+    }
+}
+// Public diff core API surface
+exports.diffCore = {
+    /**
+     * Compare two run reports and return diff result
+     */
+    diffRunReports: compareReports,
+    /**
+     * Classify the type of change between two specs
+     */
+    classifyChange: classifyDiff,
+    /**
+     * Calculate summary statistics for a diff
+     */
+    summarizeDiff: calculateDiffSummary,
+    /**
+     * Calculate deltas between two spec results
+     */
+    calculateDeltas,
+};