@pauly4010/evalai-sdk 1.8.0 → 1.9.1

package/dist/cli/ci.js

@@ -0,0 +1,192 @@
+"use strict";
+/**
+ * UX-401: One-command CI loop (evalai ci)
+ *
+ * Provides a single command teams put in .github/workflows/* and never think about again.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runCI = runCI;
+exports.runCICLI = runCICLI;
+const fs = __importStar(require("node:fs/promises"));
+const diff_1 = require("./diff");
+const discover_1 = require("./discover");
+const impact_analysis_1 = require("./impact-analysis");
+const run_1 = require("./run");
+const workspace_1 = require("./workspace");
+/**
+ * Run CI command
+ */
+async function runCI(options, projectRoot = process.cwd()) {
+    const workspace = (0, workspace_1.resolveEvalWorkspace)(projectRoot);
+    const narrative = [];
+    try {
+        // 1. Ensure .evalai workspace exists
+        await fs.mkdir(workspace.evalaiDir, { recursive: true });
+        narrative.push("✅ workspace ok");
+        // 2. Ensure manifest exists (build if missing)
+        let manifestExists = true;
+        try {
+            await fs.access(workspace.manifestPath);
+        }
+        catch {
+            manifestExists = false;
+        }
+        if (!manifestExists) {
+            console.log("📋 Building evaluation manifest...");
+            await (0, discover_1.discoverSpecs)({ manifest: true });
+            narrative.push("→ manifest built");
+        }
+        else {
+            narrative.push("→ manifest ok");
+        }
+        // 3. Run impact analysis if --impacted-only
+        let impactedSpecCount;
+        if (options.impactedOnly) {
+            const impactResult = await (0, impact_analysis_1.runImpactAnalysis)({
+                baseBranch: options.base || "main",
+            }, projectRoot);
+            impactedSpecCount = impactResult.metadata.impactedCount;
+            narrative.push(`→ impacted specs ${impactedSpecCount}`);
+        }
+        else {
+            narrative.push("→ running all specs");
+        }
+        // 4. Run evaluations
+        const runResult = await (0, run_1.runEvaluations)({
+            impactedOnly: options.impactedOnly,
+            baseBranch: options.base,
+            writeResults: options.writeResults ?? true, // Always write results for CI
+        }, projectRoot);
+        narrative.push(`→ runId ${runResult.runId}`);
+        // 5. Run diff if --base provided
+        let diffResult;
+        if (options.base) {
+            diffResult = await (0, diff_1.runDiff)({
+                base: options.base,
+                head: "last",
+            });
+            if (diffResult.summary.regressions > 0) {
+                narrative.push(`→ diff ${diffResult.summary.regressions} regressions`);
+                return {
+                    success: false,
+                    exitCode: 1,
+                    narrative: narrative.join(" "),
+                    runResult,
+                    diffResult,
+                };
+            }
+            else {
+                narrative.push("→ diff clean");
+            }
+        }
+        else {
+            narrative.push("→ no diff");
+        }
+        // 6. Check for run failures
+        if (runResult.summary.failed > 0) {
+            return {
+                success: false,
+                exitCode: 1,
+                narrative: narrative.join(" "),
+                runResult,
+                diffResult,
+            };
+        }
+        return {
+            success: true,
+            exitCode: 0,
+            narrative: narrative.join(" "),
+            runResult,
+            diffResult,
+        };
+    }
+    catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        // Print next step for debugging
+        printNextStep(errorMessage, options, workspace);
+        return {
+            success: false,
+            exitCode: 2, // Config/infra issue
+            narrative: narrative.join(" "),
+            error: errorMessage,
+        };
+    }
+}
+/**
+ * Print copy/paste debug flow
+ */
+function printNextStep(error, options, workspace) {
+    console.log("\n🔧 Next step for debugging:");
+    if (error.includes("No evaluation manifest found")) {
+        console.log("   evalai discover --manifest");
+    }
+    else if (error.includes("Base run report not found in CI environment")) {
+        console.log(`   Download base artifact and run: evalai diff --base .evalai/base-run.json --head ${workspace.lastRunPath}`);
+    }
+    else if (options.base && error.includes("Base run report not found")) {
+        console.log(`   evalai explain --report ${workspace.lastRunPath}`);
+    }
+    else {
+        console.log(`   evalai explain --report ${workspace.lastRunPath}`);
+    }
+    console.log(`   Artifacts: ${workspace.runsDir}/`);
+}
+/**
+ * CLI entry point
+ */
+async function runCICLI(options) {
+    const result = await runCI(options);
+    // Print narrative
+    console.log(`🤖 ${result.narrative}`);
+    // Print detailed results if not clean
+    if (!result.success && result.runResult) {
+        console.log("\n📊 Run Results:");
+        console.log(`   ✅ Passed: ${result.runResult.summary.passed}`);
+        console.log(`   ❌ Failed: ${result.runResult.summary.failed}`);
+        console.log(`   📊 Pass Rate: ${(result.runResult.summary.passRate * 100).toFixed(1)}%`);
+    }
+    if (!result.success && result.diffResult) {
+        console.log("\n🔄 Diff Results:");
+        console.log(`   📉 Regressions: ${result.diffResult.summary.regressions}`);
+        console.log(`   📈 Improvements: ${result.diffResult.summary.improvements}`);
+        console.log(`   📊 Pass Rate Delta: ${(result.diffResult.summary.passRateDelta * 100).toFixed(1)}%`);
+    }
+    if (result.error) {
+        console.log(`\n❌ Error: ${result.error}`);
+    }
+    // Exit with appropriate code
+    process.exit(result.exitCode);
+}

package/dist/cli/config.js

@@ -43,7 +43,11 @@ exports.mergeConfigWithArgs = mergeConfigWithArgs;
 const fs = __importStar(require("node:fs"));
 const path = __importStar(require("node:path"));
 const profiles_1 = require("./profiles");
-const CONFIG_FILES = ["evalai.config.json", "evalai.config.js", "evalai.config.cjs"];
+const CONFIG_FILES = [
+    "evalai.config.json",
+    "evalai.config.js",
+    "evalai.config.cjs",
+];
 /**
  * Find config file path in directory, walking up to root
  */
@@ -113,7 +117,11 @@ function loadConfig(cwd = process.cwd()) {
             }
             for (const key of Object.keys(config.packages)) {
                 if (relNorm === key || relNorm.startsWith(`${key}/`)) {
-                    return { ...config, ...config.packages[key], packages: config.packages };
+                    return {
+                        ...config,
+                        ...config.packages[key],
+                        packages: config.packages,
+                    };
                 }
             }
         }
@@ -156,11 +164,14 @@ function mergeConfigWithArgs(config, args) {
             merged.minScore = profile.minScore;
         if (merged.maxDrop === undefined && args.maxDrop === undefined)
             merged.maxDrop = profile.maxDrop;
-        if (merged.warnDrop === undefined && args.warnDrop === undefined && "warnDrop" in profile)
+        if (merged.warnDrop === undefined &&
+            args.warnDrop === undefined &&
+            "warnDrop" in profile)
             merged.warnDrop = profile.warnDrop;
         if (merged.minN === undefined && args.minN === undefined)
             merged.minN = profile.minN;
-        if (merged.allowWeakEvidence === undefined && args.allowWeakEvidence === undefined)
+        if (merged.allowWeakEvidence === undefined &&
+            args.allowWeakEvidence === undefined)
             merged.allowWeakEvidence = profile.allowWeakEvidence;
     }
     // Args override
@@ -172,18 +183,27 @@ function mergeConfigWithArgs(config, args) {
     }
     if (args.minScore !== undefined) {
         merged.minScore =
-            typeof args.minScore === "number" ? args.minScore : parseInt(String(args.minScore), 10);
+            typeof args.minScore === "number"
+                ? args.minScore
+                : parseInt(String(args.minScore), 10);
     }
     if (args.maxDrop !== undefined) {
         merged.maxDrop =
-            typeof args.maxDrop === "number" ? args.maxDrop : parseInt(String(args.maxDrop), 10);
+            typeof args.maxDrop === "number"
+                ? args.maxDrop
+                : parseInt(String(args.maxDrop), 10);
     }
     if (args.warnDrop !== undefined) {
         merged.warnDrop =
-            typeof args.warnDrop === "number" ? args.warnDrop : parseInt(String(args.warnDrop), 10);
+            typeof args.warnDrop === "number"
+                ? args.warnDrop
+                : parseInt(String(args.warnDrop), 10);
     }
     if (args.minN !== undefined) {
-        merged.minN = typeof args.minN === "number" ? args.minN : parseInt(String(args.minN), 10);
+        merged.minN =
+            typeof args.minN === "number"
+                ? args.minN
+                : parseInt(String(args.minN), 10);
     }
     if (args.allowWeakEvidence !== undefined) {
         merged.allowWeakEvidence =

package/dist/cli/diff.d.ts

@@ -0,0 +1,173 @@
+/**
+ * TICKET 5 — Behavioral Diff CLI (EVAL-401)
+ *
+ * Goal: "Git diff for AI behavior" from two RunReports
+ *
+ * Command:
+ * evalai diff --base main (default uses git to find baseline run)
+ * evalai diff --a <runReportPath> --b <runReportPath>
+ * evalai diff main..feature (nice-to-have alias)
+ */
+import type { RunResult } from "./run";
+/**
+ * Diff schema version
+ */
+export declare const DIFF_SCHEMA_VERSION = 1;
+/**
+ * Supported RunReport schema versions
+ */
+export declare const SUPPORTED_SCHEMA_VERSIONS: readonly [1];
+/**
+ * Rounding helpers for floating point normalization
+ */
+export declare function round(value: number, precision?: number): number;
+export declare function roundPct(value: number, precision?: number): number;
+/**
+ * Validate RunReport schema version
+ */
+export declare function validateSchemaVersion(report: RunResult): void;
+/**
+ * Diff result classification
+ */
+export type DiffClassification = "new_failure" | "fixed_failure" | "score_drop" | "score_improve" | "execution_error" | "skipped_change" | "added" | "removed";
+/**
+ * Individual spec diff
+ */
+export interface SpecDiff {
+    /** Spec identifier */
+    specId: string;
+    /** Spec name */
+    name: string;
+    /** File path */
+    filePath: string;
+    /** Classification of change */
+    classification: DiffClassification;
+    /** Base run result (if exists) */
+    base?: {
+        status: "passed" | "failed" | "skipped";
+        score?: number;
+        duration: number;
+        error?: string;
+    };
+    /** Head run result (if exists) */
+    head?: {
+        status: "passed" | "failed" | "skipped";
+        score?: number;
+        duration: number;
+        error?: string;
+    };
+    /** Calculated deltas */
+    deltas: {
+        scoreDelta?: number;
+        durationDelta?: number;
+        statusChange?: string;
+    };
+}
+/**
+ * Diff summary statistics
+ */
+export interface DiffSummary {
+    /** Total specs in base */
+    baseTotal: number;
+    /** Total specs in head */
+    headTotal: number;
+    /** Pass rate delta */
+    passRateDelta: number;
+    /** Score delta (average) */
+    scoreDelta: number;
+    /** Number of regressions */
+    regressions: number;
+    /** Number of improvements */
+    improvements: number;
+    /** Number of added specs */
+    added: number;
+    /** Number of removed specs */
+    removed: number;
+}
+/**
+ * Complete diff result
+ */
+export interface DiffResult {
+    /** Schema version */
+    schemaVersion: number;
+    /** Base run report */
+    base: RunResult;
+    /** Head run report */
+    head: RunResult;
+    /** Diff summary */
+    summary: DiffSummary;
+    /** Individual spec diffs */
+    changedSpecs: SpecDiff[];
+    /** Diff metadata */
+    metadata: {
+        generatedAt: number;
+        baseSource: string;
+        headSource: string;
+    };
+}
+/**
+ * Diff options
+ */
+export interface DiffOptions {
+    /** Base report path or branch */
+    base?: string;
+    /** Head report path */
+    head?: string;
+    /** Output format */
+    format?: "human" | "json";
+}
+/**
+ * Run diff comparison
+ */
+export declare function runDiff(options: DiffOptions): Promise<DiffResult>;
+/**
+ * Compare two run reports
+ */
+export declare function compareReports(base: RunResult, head: RunResult): DiffResult;
+/**
+ * Classify the type of change
+ */
+declare function classifyDiff(base?: RunResult["results"][0], head?: RunResult["results"][0]): DiffClassification;
+/**
+ * Calculate deltas between base and head
+ */
+declare function calculateDeltas(base?: RunResult["results"][0], head?: RunResult["results"][0]): SpecDiff["deltas"];
+/**
+ * Calculate diff summary statistics
+ */
+export declare function calculateDiffSummary(base: RunResult, head: RunResult, changedSpecs: SpecDiff[]): DiffSummary;
+/**
+ * Print human-readable diff results
+ */
+export declare function printHumanResults(result: DiffResult): void;
+/**
+ * Print JSON results
+ */
+export declare function printJsonResults(result: DiffResult): void;
+/**
+ * Write GitHub Step Summary
+ */
+export declare function writeGitHubStepSummary(result: DiffResult): Promise<void>;
+/**
+ * CLI entry point
+ */
+export declare function runDiffCLI(options: DiffOptions): Promise<void>;
+export { classifyDiff, calculateDeltas };
+export declare const diffCore: {
+    /**
+     * Compare two run reports and return diff result
+     */
+    readonly diffRunReports: typeof compareReports;
+    /**
+     * Classify the type of change between two specs
+     */
+    readonly classifyChange: typeof classifyDiff;
+    /**
+     * Calculate summary statistics for a diff
+     */
+    readonly summarizeDiff: typeof calculateDiffSummary;
+    /**
+     * Calculate deltas between two spec results
+     */
+    readonly calculateDeltas: typeof calculateDeltas;
+};