@pauly4010/evalai-sdk 1.8.0 → 1.9.0

package/CHANGELOG.md

@@ -5,6 +5,60 @@ All notable changes to the @pauly4010/evalai-sdk package will be documented in t
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.9.0] - 2026-02-27
+
+### ✨ Added
+
+#### CLI — One-Command CI Loop (`evalai ci`)
+
+- **`evalai ci`** — Single command teams put in GitHub workflows and never think about again
+- **Complete CI pipeline**: discover → manifest → impact → run → diff → PR summary → safe failure → "next step"
+- **Automatic manifest building**: Builds manifest if missing, no manual steps required
+- **Impact analysis integration**: `--impacted-only` flag for targeted testing
+- **Smart exit codes**: 0=clean, 1=regressions, 2=config/infra issues
+- **Self-documenting failures**: Always prints copy/paste next step for debugging
+- **GitHub Step Summary integration**: Automatic PR summaries with regressions and artifacts
+
+#### CLI — Durable Run History & Diff System
+
+- **Run artifact retention**: Timestamped artifacts in `.evalai/runs/run-<runId>.json`
+- **Run index file**: `.evalai/runs/index.json` tracks all runs with metadata
+- **Schema versioning**: `RunResult` and `DiffResult` include `schemaVersion` for compatibility
+- **Base/head shortcuts**: `--base baseline`, `--base last`, `--head last` for common cases
+- **Floating point normalization**: Consistent score/delta calculations across runs
+- **Comprehensive diff comparison**: Classifies regressions, improvements, added, removed specs
+
+#### CLI — Centralized Architecture
+
+- **Environment detection**: `isCI()`, `isGitHubActions()`, `getGitHubStepSummaryPath()` unified
+- **Workspace resolution**: `resolveEvalWorkspace()` provides all `.evalai` paths
+- **Git reference detection**: Comprehensive patterns for branches, tags, and ranges
+- **No more duplication**: All commands use shared utilities for consistency
+
+#### CLI — CI Friendliness
+
+- **Fail-safe base resolution**: Clear error messages when base artifacts missing in CI
+- **GitHub Step Summary**: Rich markdown summaries with metrics, regressions, and artifact links
+- **CI-specific error handling**: Exit code 2 for config issues with helpful guidance
+- **Artifact download instructions**: Exact commands for manual base artifact setup
+
+### 🔧 Changed
+
+- **Exit codes standardized**: 0=clean, 1=regressions, 2=config/infra issues across all commands
+- **Schema compatibility**: Added `schemaVersion` validation for future-proofing
+- **Path resolution**: All commands use centralized workspace helpers
+- **Error messages**: More actionable and context-aware guidance
+
+### 📊 New Features Summary
+
+- **One-command CI**: `evalai ci` replaces multi-step workflows
+- **Durable history**: Run artifacts preserved with smart indexing
+- **Smart diffing**: Automated regression detection with GitHub integration
+- **Centralized utilities**: Environment detection and workspace resolution unified
+- **Self-documenting**: Clear next steps for any failure scenario
+
+---
+
 ## [1.8.0] - 2026-02-26
 
 ### ✨ Added

package/dist/cli/ci.d.ts

@@ -0,0 +1,45 @@
+/**
+ * UX-401: One-command CI loop (evalai ci)
+ *
+ * Provides a single command teams put in .github/workflows/* and never think about again.
+ */
+import type { DiffResult } from "./diff";
+import type { RunResult } from "./run";
+/**
+ * CI command options
+ */
+export interface CIOptions {
+    /** Base reference for diff comparison */
+    base?: string;
+    /** Run only impacted specs */
+    impactedOnly?: boolean;
+    /** Output format */
+    format?: "human" | "json" | "github";
+    /** Write run results */
+    writeResults?: boolean;
+}
+/**
+ * CI execution result
+ */
+export interface CIResult {
+    /** Success status */
+    success: boolean;
+    /** Exit code */
+    exitCode: number;
+    /** Execution narrative */
+    narrative: string;
+    /** Run result (if executed) */
+    runResult?: RunResult;
+    /** Diff result (if executed) */
+    diffResult?: DiffResult;
+    /** Error message (if failed) */
+    error?: string;
+}
+/**
+ * Run CI command
+ */
+export declare function runCI(options: CIOptions, projectRoot?: string): Promise<CIResult>;
+/**
+ * CLI entry point
+ */
+export declare function runCICLI(options: CIOptions): Promise<void>;

package/dist/cli/ci.js

@@ -0,0 +1,192 @@
+"use strict";
+/**
+ * UX-401: One-command CI loop (evalai ci)
+ *
+ * Provides a single command teams put in .github/workflows/* and never think about again.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runCI = runCI;
+exports.runCICLI = runCICLI;
+const fs = __importStar(require("node:fs/promises"));
+const diff_1 = require("./diff");
+const discover_1 = require("./discover");
+const impact_analysis_1 = require("./impact-analysis");
+const run_1 = require("./run");
+const workspace_1 = require("./workspace");
+/**
+ * Run CI command
+ */
+async function runCI(options, projectRoot = process.cwd()) {
+    const workspace = (0, workspace_1.resolveEvalWorkspace)(projectRoot);
+    const narrative = [];
+    try {
+        // 1. Ensure .evalai workspace exists
+        await fs.mkdir(workspace.evalaiDir, { recursive: true });
+        narrative.push("✅ workspace ok");
+        // 2. Ensure manifest exists (build if missing)
+        let manifestExists = true;
+        try {
+            await fs.access(workspace.manifestPath);
+        }
+        catch {
+            manifestExists = false;
+        }
+        if (!manifestExists) {
+            console.log("📋 Building evaluation manifest...");
+            await (0, discover_1.discoverSpecs)({ manifest: true });
+            narrative.push("→ manifest built");
+        }
+        else {
+            narrative.push("→ manifest ok");
+        }
+        // 3. Run impact analysis if --impacted-only
+        let impactedSpecCount;
+        if (options.impactedOnly) {
+            const impactResult = await (0, impact_analysis_1.runImpactAnalysis)({
+                baseBranch: options.base || "main",
+            }, projectRoot);
+            impactedSpecCount = impactResult.metadata.impactedCount;
+            narrative.push(`→ impacted specs ${impactedSpecCount}`);
+        }
+        else {
+            narrative.push("→ running all specs");
+        }
+        // 4. Run evaluations
+        const runResult = await (0, run_1.runEvaluations)({
+            impactedOnly: options.impactedOnly,
+            baseBranch: options.base,
+            writeResults: options.writeResults ?? true, // Always write results for CI
+        }, projectRoot);
+        narrative.push(`→ runId ${runResult.runId}`);
+        // 5. Run diff if --base provided
+        let diffResult;
+        if (options.base) {
+            diffResult = await (0, diff_1.runDiff)({
+                base: options.base,
+                head: "last",
+            });
+            if (diffResult.summary.regressions > 0) {
+                narrative.push(`→ diff ${diffResult.summary.regressions} regressions`);
+                return {
+                    success: false,
+                    exitCode: 1,
+                    narrative: narrative.join(" "),
+                    runResult,
+                    diffResult,
+                };
+            }
+            else {
+                narrative.push("→ diff clean");
+            }
+        }
+        else {
+            narrative.push("→ no diff");
+        }
+        // 6. Check for run failures
+        if (runResult.summary.failed > 0) {
+            return {
+                success: false,
+                exitCode: 1,
+                narrative: narrative.join(" "),
+                runResult,
+                diffResult,
+            };
+        }
+        return {
+            success: true,
+            exitCode: 0,
+            narrative: narrative.join(" "),
+            runResult,
+            diffResult,
+        };
+    }
+    catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        // Print next step for debugging
+        printNextStep(errorMessage, options, workspace);
+        return {
+            success: false,
+            exitCode: 2, // Config/infra issue
+            narrative: narrative.join(" "),
+            error: errorMessage,
+        };
+    }
+}
+/**
+ * Print copy/paste debug flow
+ */
+function printNextStep(error, options, workspace) {
+    console.log("\n🔧 Next step for debugging:");
+    if (error.includes("No evaluation manifest found")) {
+        console.log("   evalai discover --manifest");
+    }
+    else if (error.includes("Base run report not found in CI environment")) {
+        console.log(`   Download base artifact and run: evalai diff --base .evalai/base-run.json --head ${workspace.lastRunPath}`);
+    }
+    else if (options.base && error.includes("Base run report not found")) {
+        console.log(`   evalai explain --report ${workspace.lastRunPath}`);
+    }
+    else {
+        console.log(`   evalai explain --report ${workspace.lastRunPath}`);
+    }
+    console.log(`   Artifacts: ${workspace.runsDir}/`);
+}
+/**
+ * CLI entry point
+ */
+async function runCICLI(options) {
+    const result = await runCI(options);
+    // Print narrative
+    console.log(`🤖 ${result.narrative}`);
+    // Print detailed results if not clean
+    if (!result.success && result.runResult) {
+        console.log("\n📊 Run Results:");
+        console.log(`   ✅ Passed: ${result.runResult.summary.passed}`);
+        console.log(`   ❌ Failed: ${result.runResult.summary.failed}`);
+        console.log(`   📊 Pass Rate: ${(result.runResult.summary.passRate * 100).toFixed(1)}%`);
+    }
+    if (!result.success && result.diffResult) {
+        console.log("\n🔄 Diff Results:");
+        console.log(`   📉 Regressions: ${result.diffResult.summary.regressions}`);
+        console.log(`   📈 Improvements: ${result.diffResult.summary.improvements}`);
+        console.log(`   📊 Pass Rate Delta: ${(result.diffResult.summary.passRateDelta * 100).toFixed(1)}%`);
+    }
+    if (result.error) {
+        console.log(`\n❌ Error: ${result.error}`);
+    }
+    // Exit with appropriate code
+    process.exit(result.exitCode);
+}

package/dist/cli/diff.d.ts

@@ -0,0 +1,173 @@
+/**
+ * TICKET 5 — Behavioral Diff CLI (EVAL-401)
+ *
+ * Goal: "Git diff for AI behavior" from two RunReports
+ *
+ * Command:
+ * evalai diff --base main (default uses git to find baseline run)
+ * evalai diff --a <runReportPath> --b <runReportPath>
+ * evalai diff main..feature (nice-to-have alias)
+ */
+import type { RunResult } from "./run";
+/**
+ * Diff schema version
+ */
+export declare const DIFF_SCHEMA_VERSION = 1;
+/**
+ * Supported RunReport schema versions
+ */
+export declare const SUPPORTED_SCHEMA_VERSIONS: readonly [1];
+/**
+ * Rounding helpers for floating point normalization
+ */
+export declare function round(value: number, precision?: number): number;
+export declare function roundPct(value: number, precision?: number): number;
+/**
+ * Validate RunReport schema version
+ */
+export declare function validateSchemaVersion(report: RunResult): void;
+/**
+ * Diff result classification
+ */
+export type DiffClassification = "new_failure" | "fixed_failure" | "score_drop" | "score_improve" | "execution_error" | "skipped_change" | "added" | "removed";
+/**
+ * Individual spec diff
+ */
+export interface SpecDiff {
+    /** Spec identifier */
+    specId: string;
+    /** Spec name */
+    name: string;
+    /** File path */
+    filePath: string;
+    /** Classification of change */
+    classification: DiffClassification;
+    /** Base run result (if exists) */
+    base?: {
+        status: "passed" | "failed" | "skipped";
+        score?: number;
+        duration: number;
+        error?: string;
+    };
+    /** Head run result (if exists) */
+    head?: {
+        status: "passed" | "failed" | "skipped";
+        score?: number;
+        duration: number;
+        error?: string;
+    };
+    /** Calculated deltas */
+    deltas: {
+        scoreDelta?: number;
+        durationDelta?: number;
+        statusChange?: string;
+    };
+}
+/**
+ * Diff summary statistics
+ */
+export interface DiffSummary {
+    /** Total specs in base */
+    baseTotal: number;
+    /** Total specs in head */
+    headTotal: number;
+    /** Pass rate delta */
+    passRateDelta: number;
+    /** Score delta (average) */
+    scoreDelta: number;
+    /** Number of regressions */
+    regressions: number;
+    /** Number of improvements */
+    improvements: number;
+    /** Number of added specs */
+    added: number;
+    /** Number of removed specs */
+    removed: number;
+}
+/**
+ * Complete diff result
+ */
+export interface DiffResult {
+    /** Schema version */
+    schemaVersion: number;
+    /** Base run report */
+    base: RunResult;
+    /** Head run report */
+    head: RunResult;
+    /** Diff summary */
+    summary: DiffSummary;
+    /** Individual spec diffs */
+    changedSpecs: SpecDiff[];
+    /** Diff metadata */
+    metadata: {
+        generatedAt: number;
+        baseSource: string;
+        headSource: string;
+    };
+}
+/**
+ * Diff options
+ */
+export interface DiffOptions {
+    /** Base report path or branch */
+    base?: string;
+    /** Head report path */
+    head?: string;
+    /** Output format */
+    format?: "human" | "json";
+}
+/**
+ * Run diff comparison
+ */
+export declare function runDiff(options: DiffOptions): Promise<DiffResult>;
+/**
+ * Compare two run reports
+ */
+export declare function compareReports(base: RunResult, head: RunResult): DiffResult;
+/**
+ * Classify the type of change
+ */
+declare function classifyDiff(base?: RunResult["results"][0], head?: RunResult["results"][0]): DiffClassification;
+/**
+ * Calculate deltas between base and head
+ */
+declare function calculateDeltas(base?: RunResult["results"][0], head?: RunResult["results"][0]): SpecDiff["deltas"];
+/**
+ * Calculate diff summary statistics
+ */
+export declare function calculateDiffSummary(base: RunResult, head: RunResult, changedSpecs: SpecDiff[]): DiffSummary;
+/**
+ * Print human-readable diff results
+ */
+export declare function printHumanResults(result: DiffResult): void;
+/**
+ * Print JSON results
+ */
+export declare function printJsonResults(result: DiffResult): void;
+/**
+ * Write GitHub Step Summary
+ */
+export declare function writeGitHubStepSummary(result: DiffResult): Promise<void>;
+/**
+ * CLI entry point
+ */
+export declare function runDiffCLI(options: DiffOptions): Promise<void>;
+export { classifyDiff, calculateDeltas };
+export declare const diffCore: {
+    /**
+     * Compare two run reports and return diff result
+     */
+    readonly diffRunReports: typeof compareReports;
+    /**
+     * Classify the type of change between two specs
+     */
+    readonly classifyChange: typeof classifyDiff;
+    /**
+     * Calculate summary statistics for a diff
+     */
+    readonly summarizeDiff: typeof calculateDiffSummary;
+    /**
+     * Calculate deltas between two spec results
+     */
+    readonly calculateDeltas: typeof calculateDeltas;
+};