@pauly4010/evalai-sdk 1.8.0 → 1.9.1

package/dist/cli/impact-analysis.js

@@ -0,0 +1,252 @@
+"use strict";
+/**
+ * TICKET 3 — Impact Analysis CLI Command (v0)
+ *
+ * Goal: Modal-like perceived speed via incremental intelligence
+ *
+ * Algorithm v0 (practical, shippable):
+ * - Inputs: manifest.json + git diff --name-only base...HEAD
+ * - Rules: Direct file mapping, dependency tracking, safe fallback
+ * - Output: Human-readable counts + JSON for automation
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runImpactAnalysis = runImpactAnalysis;
+exports.analyzeImpact = analyzeImpact;
+exports.printHumanResults = printHumanResults;
+exports.printJsonResults = printJsonResults;
+exports.runImpactAnalysisCLI = runImpactAnalysisCLI;
+const node_child_process_1 = require("node:child_process");
+const fs = __importStar(require("node:fs/promises"));
+const path = __importStar(require("node:path"));
+/**
+ * Run impact analysis
+ */
+async function runImpactAnalysis(options, projectRoot = process.cwd()) {
+    const startTime = Date.now();
+    // Read manifest
+    const manifest = await readManifest(projectRoot);
+    if (!manifest) {
+        throw new Error("No evaluation manifest found. Run 'evalai discover --manifest' first.");
+    }
+    // Get changed files
+    const changedFiles = options.changedFiles || (await getChangedFiles(options.baseBranch));
+    // Analyze impact
+    const { impactedSpecIds, reasonBySpecId } = analyzeImpact(changedFiles, manifest);
+    const result = {
+        impactedSpecIds,
+        reasonBySpecId,
+        changedFiles,
+        metadata: {
+            baseBranch: options.baseBranch,
+            totalSpecs: manifest.specs.length,
+            impactedCount: impactedSpecIds.length,
+            analysisTime: Date.now() - startTime,
+        },
+    };
+    return result;
+}
+/**
+ * Read evaluation manifest
+ */
+async function readManifest(projectRoot = process.cwd()) {
+    const manifestPath = path.join(projectRoot, ".evalai", "manifest.json");
+    try {
+        const content = await fs.readFile(manifestPath, "utf-8");
+        return JSON.parse(content);
+    }
+    catch (_error) {
+        return null;
+    }
+}
+/**
+ * Get changed files from git
+ */
+async function getChangedFiles(baseBranch) {
+    return new Promise((resolve, reject) => {
+        const git = (0, node_child_process_1.spawn)("git", ["diff", "--name-only", `${baseBranch}...HEAD`], {
+            stdio: ["pipe", "pipe", "pipe"],
+        });
+        let output = "";
+        let error = "";
+        git.stdout?.on("data", (data) => {
+            output += data.toString();
+        });
+        git.stderr?.on("data", (data) => {
+            error += data.toString();
+        });
+        git.on("close", (code) => {
+            if (code !== 0) {
+                reject(new Error(`Git diff failed: ${error}`));
+                return;
+            }
+            const files = output
+                .split("\n")
+                .map((f) => f.trim())
+                .filter((f) => f.length > 0)
+                .map((f) => f.replace(/\\/g, "/")); // Normalize to POSIX
+            resolve(files);
+        });
+    });
+}
+/**
+ * Analyze impact of changed files
+ */
+function analyzeImpact(changedFiles, manifest) {
+    const impactedSpecIds = new Set();
+    const reasonBySpecId = {};
+    // Normalize changed files to POSIX format
+    const normalizedChangedFiles = changedFiles.map((f) => f.replace(/\\/g, "/"));
+    // Create lookup maps
+    const specsByFile = new Map();
+    const specsByDependency = new Map();
+    // Index specs by file
+    for (const spec of manifest.specs) {
+        // By file path
+        if (!specsByFile.has(spec.filePath)) {
+            specsByFile.set(spec.filePath, []);
+        }
+        specsByFile.get(spec.filePath)?.push(spec);
+        // By dependencies
+        const deps = [
+            ...spec.dependsOn.prompts,
+            ...spec.dependsOn.datasets,
+            ...spec.dependsOn.tools,
+            ...spec.dependsOn.code,
+        ];
+        for (const dep of deps) {
+            if (!specsByDependency.has(dep)) {
+                specsByDependency.set(dep, []);
+            }
+            specsByDependency.get(dep)?.push(spec);
+        }
+    }
+    // Analyze each changed file
+    for (const changedFile of normalizedChangedFiles) {
+        // Rule 1: Direct spec file change
+        const specsInFile = specsByFile.get(changedFile);
+        if (specsInFile) {
+            for (const spec of specsInFile) {
+                impactedSpecIds.add(spec.id);
+                reasonBySpecId[spec.id] = `Spec file changed: ${changedFile}`;
+            }
+        }
+        // Rule 2: Dependency change
+        const specsUsingDep = specsByDependency.get(changedFile);
+        if (specsUsingDep) {
+            for (const spec of specsUsingDep) {
+                impactedSpecIds.add(spec.id);
+                reasonBySpecId[spec.id] = `Dependency changed: ${changedFile}`;
+            }
+        }
+        // Rule 3: Safe fallback for unknown files
+        if (!specsInFile && !specsUsingDep) {
+            // If we can't map the file, be conservative and run everything
+            console.warn(`⚠️  Unknown changed file: ${changedFile}`);
+            console.warn(`🛡️  Running full suite for safety`);
+            // Add all specs
+            for (const spec of manifest.specs) {
+                impactedSpecIds.add(spec.id);
+                reasonBySpecId[spec.id] =
+                    `Unknown file changed: ${changedFile} (safe fallback)`;
+            }
+            break; // No need to continue analyzing
+        }
+    }
+    return {
+        impactedSpecIds: Array.from(impactedSpecIds).sort(),
+        reasonBySpecId,
+    };
+}
+/**
+ * Print human-readable results
+ */
+function printHumanResults(result) {
+    console.log("\n🔍 Impact Analysis Results");
+    console.log(`📊 Base branch: ${result.metadata.baseBranch}`);
+    console.log(`📁 Changed files: ${result.changedFiles.length}`);
+    console.log(`🎯 Impacted specs: ${result.metadata.impactedCount}/${result.metadata.totalSpecs}`);
+    console.log(`⏱️  Analysis time: ${result.metadata.analysisTime}ms`);
+    if (result.changedFiles.length > 0) {
+        console.log("\n📝 Changed files:");
+        for (const file of result.changedFiles) {
+            console.log(`   • ${file}`);
+        }
+    }
+    if (result.impactedSpecIds.length > 0) {
+        console.log("\n🎯 Impacted specifications:");
+        for (const specId of result.impactedSpecIds) {
+            const reason = result.reasonBySpecId[specId];
+            console.log(`   • ${specId} (${reason})`);
+        }
+        console.log("\n💡 Suggested command:");
+        console.log(`   evalai run --spec-ids ${result.impactedSpecIds.join(",")}`);
+    }
+    else {
+        console.log("\n✅ No specifications impacted");
+        console.log("💡 No tests needed to run");
+    }
+}
+/**
+ * Print JSON results
+ */
+function printJsonResults(result) {
+    console.log(JSON.stringify(result, null, 2));
+}
+/**
+ * CLI entry point
+ */
+async function runImpactAnalysisCLI(options) {
+    try {
+        const result = await runImpactAnalysis(options);
+        if (options.format === "json") {
+            printJsonResults(result);
+        }
+        else {
+            printHumanResults(result);
+        }
+        // Exit with appropriate code
+        if (result.metadata.impactedCount === 0) {
+            process.exit(0);
+        }
+        else {
+            process.exit(1); // Signal that tests should run
+        }
+    }
+    catch (error) {
+        console.error("❌ Impact analysis failed:", error instanceof Error ? error.message : String(error));
+        process.exit(2);
+    }
+}

package/dist/cli/index.js

@@ -10,11 +10,17 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const baseline_1 = require("./baseline");
 const check_1 = require("./check");
+const ci_1 = require("./ci");
+const diff_1 = require("./diff");
+const discover_1 = require("./discover");
 const doctor_1 = require("./doctor");
 const explain_1 = require("./explain");
+const impact_analysis_1 = require("./impact-analysis");
 const init_1 = require("./init");
+const migrate_1 = require("./migrate");
 const print_config_1 = require("./print-config");
 const regression_gate_1 = require("./regression-gate");
+const run_1 = require("./run");
 const share_1 = require("./share");
 const upgrade_1 = require("./upgrade");
 const argv = process.argv.slice(2);
@@ -32,6 +38,60 @@ else if (subcommand === "gate") {
     const code = (0, regression_gate_1.runGate)(argv.slice(1));
     process.exit(code);
 }
+else if (subcommand === "migrate") {
+    // Handle migrate subcommand
+    const migrateSubcommand = argv[1];
+    if (migrateSubcommand === "config") {
+        // Parse migrate config arguments
+        let inputPath = "";
+        let outputPath = "";
+        let verbose = false;
+        let helpers = true;
+        let preserveIds = true;
+        let provenance = true;
+        for (let i = 2; i < argv.length; i++) {
+            const arg = argv[i];
+            if (arg === "--in" || arg === "-i") {
+                inputPath = argv[++i];
+            }
+            else if (arg === "--out" || arg === "-o") {
+                outputPath = argv[++i];
+            }
+            else if (arg === "--verbose" || arg === "-v") {
+                verbose = true;
+            }
+            else if (arg === "--no-helpers") {
+                helpers = false;
+            }
+            else if (arg === "--no-preserve-ids") {
+                preserveIds = false;
+            }
+            else if (arg === "--no-provenance") {
+                provenance = false;
+            }
+        }
+        if (!inputPath || !outputPath) {
+            console.error("Error: Both --in and --out options are required");
+            console.error("Usage: evalai migrate config --in <input> --out <output> [options]");
+            process.exit(1);
+        }
+        (0, migrate_1.migrateConfig)({
+            input: inputPath,
+            output: outputPath,
+            verbose,
+            helpers,
+            preserveIds,
+            provenance,
+        }).catch((err) => {
+            console.error(`Migration failed: ${err instanceof Error ? err.message : String(err)}`);
+            process.exit(1);
+        });
+    }
+    else {
+        console.error("Error: Unknown migrate subcommand. Use 'evalai migrate config'");
+        process.exit(1);
+    }
+}
 else if (subcommand === "upgrade") {
     const code = (0, upgrade_1.runUpgrade)(argv.slice(1));
     process.exit(code);
@@ -82,11 +142,123 @@ else if (subcommand === "share") {
         process.exit(1);
     });
 }
+else if (subcommand === "discover") {
+    // Parse arguments for discover command
+    const args = argv.slice(1);
+    const manifestFlag = args.includes("--manifest");
+    (0, discover_1.discoverSpecs)({ manifest: manifestFlag })
+        .then(() => process.exit(0))
+        .catch((err) => {
+        console.error(`EvalAI ERROR: ${err instanceof Error ? err.message : String(err)}`);
+        process.exit(1);
+    });
+}
+else if (subcommand === "impact-analysis") {
+    // Parse arguments for impact-analysis command
+    const args = argv.slice(1);
+    const baseIndex = args.indexOf("--base");
+    const changedFilesIndex = args.indexOf("--changed-files");
+    const formatIndex = args.indexOf("--format");
+    const baseBranch = baseIndex !== -1 ? args[baseIndex + 1] : "main";
+    const changedFiles = changedFilesIndex !== -1
+        ? args[changedFilesIndex + 1]?.split(",")
+        : undefined;
+    const format = formatIndex !== -1 ? args[formatIndex + 1] : "human";
+    (0, impact_analysis_1.runImpactAnalysisCLI)({ baseBranch, changedFiles, format })
+        .then(() => process.exit(0))
+        .catch((err) => {
+        console.error(`EvalAI ERROR: ${err instanceof Error ? err.message : String(err)}`);
+        process.exit(2);
+    });
+}
+else if (subcommand === "run") {
+    // Parse arguments for run command
+    const args = argv.slice(1);
+    const specIdsIndex = args.indexOf("--spec-ids");
+    const impactedOnlyIndex = args.indexOf("--impacted-only");
+    const baseIndex = args.indexOf("--base");
+    const formatIndex = args.indexOf("--format");
+    const writeResultsIndex = args.indexOf("--write-results");
+    const specIds = specIdsIndex !== -1 ? args[specIdsIndex + 1]?.split(",") : undefined;
+    const impactedOnly = impactedOnlyIndex !== -1;
+    const baseBranch = baseIndex !== -1 ? args[baseIndex + 1] : undefined;
+    const format = formatIndex !== -1 ? args[formatIndex + 1] : "human";
+    const writeResults = writeResultsIndex !== -1;
+    (0, run_1.runEvaluationsCLI)({
+        specIds,
+        impactedOnly: impactedOnly ? !!baseBranch : false,
+        baseBranch,
+        format,
+        writeResults,
+    })
+        .then(() => process.exit(0))
+        .catch((err) => {
+        console.error(`EvalAI ERROR: ${err instanceof Error ? err.message : String(err)}`);
+        process.exit(2);
+    });
+}
+else if (subcommand === "diff") {
+    // Parse arguments for diff command
+    const args = argv.slice(1);
+    const baseIndex = args.indexOf("--base");
+    const headIndex = args.indexOf("--head");
+    const formatIndex = args.indexOf("--format");
+    const base = baseIndex !== -1 ? args[baseIndex + 1] : undefined;
+    const head = headIndex !== -1 ? args[headIndex + 1] : undefined;
+    const format = formatIndex !== -1 ? args[formatIndex + 1] : "human";
+    (0, diff_1.runDiffCLI)({ base, head, format })
+        .then(() => process.exit(0))
+        .catch((err) => {
+        console.error(`EvalAI ERROR: ${err instanceof Error ? err.message : String(err)}`);
+        process.exit(2);
+    });
+}
+else if (subcommand === "ci") {
+    // Parse arguments for ci command
+    const args = argv.slice(1);
+    const baseIndex = args.indexOf("--base");
+    const impactedOnlyIndex = args.indexOf("--impacted-only");
+    const formatIndex = args.indexOf("--format");
+    const writeResultsIndex = args.indexOf("--write-results");
+    const base = baseIndex !== -1 ? args[baseIndex + 1] : undefined;
+    const impactedOnly = impactedOnlyIndex !== -1;
+    const format = formatIndex !== -1
+        ? args[formatIndex + 1]
+        : "human";
+    const writeResults = writeResultsIndex !== -1;
+    (0, ci_1.runCICLI)({ base, impactedOnly, format, writeResults })
+        .then(() => process.exit(0))
+        .catch((err) => {
+        console.error(`EvalAI ERROR: ${err instanceof Error ? err.message : String(err)}`);
+        process.exit(2);
+    });
+}
 else {
     console.log(`EvalAI CLI
 
 Usage:
   evalai init                Create evalai.config.json + baseline + CI workflow
+  evalai discover            Discover behavioral specs in project and show statistics
+  evalai discover --manifest  Generate evaluation manifest for incremental analysis
+  evalai impact-analysis     Analyze impact of changes and suggest targeted tests
+    --base <branch>          Base branch to compare against (default: main)
+    --changed-files <files>  Comma-separated list of changed files (for CI)
+    --format <fmt>           Output format: human (default), json
+  evalai ci                  One-command CI loop (manifest → impact → run → diff)
+    --base <ref>            Base reference for diff (baseline|last|<runId>|<path>|<gitref>)
+    --impacted-only          Run only specs impacted by changes
+    --format <fmt>           Output format: human (default), json, github
+    --write-results          Write run results to .evalai/last-run.json
+  evalai run                 Run evaluation specifications
+    --spec-ids <ids>         Comma-separated list of spec IDs to run
+    --impacted-only          Run only specs impacted by changes (requires --base)
+    --base <branch>          Base branch for impact analysis (with --impacted-only)
+    --format <fmt>           Output format: human (default), json
+    --write-results          Write results to .evalai/last-run.json
+  evalai diff                Compare two run reports and show behavioral changes
+    --base <branch>          Base branch or report path (default: main)
+    --head <path>            Head report path (default: .evalai/last-run.json)
+    --format <fmt>           Output format: human (default), json
   evalai gate [options]      Run regression gate (local test-based, no API needed)
   evalai check [options]     CI/CD evaluation gate (API-based)
   evalai explain [options]   Explain last gate/check failure with root causes + fixes
@@ -133,6 +305,19 @@ Options for doctor:
 
 Examples:
   evalai init
+  evalai discover
+  evalai discover --manifest
+  evalai impact-analysis --base main
+  evalai impact-analysis --base main --format json
+  evalai impact-analysis --changed-files src/utils.ts,datasets/test.json
+  evalai run
+  evalai run --spec-ids spec1,spec2
+  evalai run --impacted-only --base main
+  evalai run --format json --write-results
+  evalai diff
+  evalai diff --base main
+  evalai diff --base main --format json
+  evalai diff --a .evalai/runs/base.json --b .evalai/last-run.json
   evalai gate
   evalai gate --format json
   evalai explain

package/dist/cli/manifest.d.ts

@@ -0,0 +1,103 @@
+/**
+ * TICKET 2 — Evaluation Manifest Generation
+ *
+ * Goal: turn discovery output into a stable, versioned, machine-consumable artifact
+ * that becomes the input to run / impact / diff.
+ *
+ * This is the compiler output that everything else consumes.
+ */
+import type { ExecutionModeConfig } from "../runtime/execution-mode";
+import { SDK_VERSION } from "../version";
+import type { SpecAnalysis } from "./discover";
+export { SDK_VERSION };
+/**
+ * Manifest schema version
+ */
+export declare const MANIFEST_SCHEMA_VERSION = 1;
+/**
+ * Evaluation Manifest Schema
+ */
+export interface EvaluationManifest {
+    /** Schema version for compatibility */
+    schemaVersion: number;
+    /** When this manifest was generated */
+    generatedAt: number;
+    /** Project metadata */
+    project: {
+        name: string;
+        root: string;
+        namespace: string;
+    };
+    /** Runtime information */
+    runtime: {
+        mode: "spec" | "legacy";
+        sdkVersion: string;
+    };
+    /** Spec files with hashes */
+    specFiles: SpecFile[];
+    /** Individual specifications */
+    specs: Spec[];
+}
+/**
+ * Spec file information
+ */
+export interface SpecFile {
+    /** POSIX-relative file path */
+    filePath: string;
+    /** SHA-256 hash of file content */
+    fileHash: string;
+    /** Number of specs in this file */
+    specCount: number;
+}
+/**
+ * Individual specification
+ */
+export interface Spec {
+    /** Stable canonical ID */
+    id: string;
+    /** Spec name */
+    name: string;
+    /** Suite path from tags or file structure */
+    suitePath: string[];
+    /** POSIX-relative file path */
+    filePath: string;
+    /** Position in file */
+    position: {
+        line: number;
+        column: number;
+    };
+    /** Tags/categories */
+    tags: string[];
+    /** Dependencies */
+    dependsOn: {
+        prompts: string[];
+        datasets: string[];
+        tools: string[];
+        code: string[];
+    };
+}
+/**
+ * Lock file for caching
+ */
+export interface ManifestLock {
+    /** When lock was generated */
+    generatedAt: number;
+    /** File hashes for incremental updates */
+    fileHashes: Record<string, string>;
+}
+/**
+ * Generate evaluation manifest from discovery results
+ */
+export declare function generateManifest(specs: SpecAnalysis[], projectRoot: string, projectName: string, executionMode: ExecutionModeConfig): Promise<EvaluationManifest>;
+/**
+ * Write manifest to disk
+ */
+export declare function writeManifest(manifest: EvaluationManifest, projectRoot: string): Promise<void>;
+/**
+ * Read existing manifest
+ */
+export declare function readManifest(projectRoot: string): Promise<EvaluationManifest | null>;
+/**
+ * Read existing lock file
+ */
+export declare function readLock(projectRoot: string): Promise<ManifestLock | null>;