@pauly4010/evalai-sdk 1.6.0 → 1.7.0

package/dist/cli/init.js

@@ -1,9 +1,18 @@
 #!/usr/bin/env node
 "use strict";
 /**
- * evalai init — Create evalai.config.json
+ * evalai init — Full project scaffolder
  *
- * Creates the smallest possible config file. Defaults belong in code.
+ * Zero-to-gate in under 5 minutes:
+ *   npx evalai init
+ *   git push
+ *   …CI starts blocking regressions.
+ *
+ * What it does:
+ *   1. Detects Node repo + package manager
+ *   2. Creates evals/ directory + baseline.json
+ *   3. Installs .github/workflows/evalai-gate.yml
+ *   4. Prints next steps (no docs required)
  */
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
@@ -40,30 +49,232 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runInit = runInit;
+const node_child_process_1 = require("node:child_process");
 const fs = __importStar(require("node:fs"));
 const path = __importStar(require("node:path"));
-const CONFIG_CONTENT = `{
-  "evaluationId": ""
+function detectProject(cwd) {
+    const pkgPath = path.join(cwd, "package.json");
+    if (!fs.existsSync(pkgPath))
+        return null;
+    let pkg;
+    try {
+        pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
+    }
+    catch {
+        return null;
+    }
+    let pm = "npm";
+    if (fs.existsSync(path.join(cwd, "pnpm-lock.yaml")))
+        pm = "pnpm";
+    else if (fs.existsSync(path.join(cwd, "yarn.lock")))
+        pm = "yarn";
+    const testScript = pkg.scripts?.test ?? "";
+    const hasTestScript = !!testScript && testScript !== 'echo "Error: no test specified" && exit 1';
+    return {
+        cwd,
+        pm,
+        hasTestScript,
+        testScript,
+        name: pkg.name ?? path.basename(cwd),
+    };
+}
+// ── Step helpers ──
+function ok(msg) {
+    console.log(`  ✔ ${msg}`);
+}
+function skip(msg) {
+    console.log(`  – ${msg}`);
+}
+// ── 1. Create evals/ + baseline.json ──
+function createBaseline(cwd, project) {
+    const evalsDir = path.join(cwd, "evals");
+    const baselinePath = path.join(evalsDir, "baseline.json");
+    if (fs.existsSync(baselinePath)) {
+        skip("evals/baseline.json already exists");
+        return true;
+    }
+    if (!fs.existsSync(evalsDir)) {
+        fs.mkdirSync(evalsDir, { recursive: true });
+    }
+    const user = process.env.USER || process.env.USERNAME || "unknown";
+    const now = new Date().toISOString();
+    // Run tests to capture real count if possible
+    let testTotal = 0;
+    let testsPassed = true;
+    if (project.hasTestScript) {
+        const isWin = process.platform === "win32";
+        const result = (0, node_child_process_1.spawnSync)(project.pm, ["test"], {
+            cwd,
+            stdio: "pipe",
+            shell: isWin,
+            timeout: 120000,
+        });
+        testsPassed = result.status === 0;
+        // Try to extract test count from output
+        const output = (result.stdout?.toString() ?? "") + (result.stderr?.toString() ?? "");
+        const countMatch = output.match(/(\d+)\s+(?:tests?|specs?)\s+(?:passed|completed)/i) ??
+            output.match(/Tests:\s+(\d+)\s+passed/i) ??
+            output.match(/(\d+)\s+passing/i);
+        if (countMatch)
+            testTotal = parseInt(countMatch[1], 10);
+    }
+    const baseline = {
+        schemaVersion: 1,
+        description: `Regression gate baseline for ${project.name}`,
+        generatedAt: now,
+        generatedBy: user,
+        commitSha: getHeadSha(cwd),
+        updatedAt: now,
+        updatedBy: user,
+        tolerance: {
+            scoreDrop: 5,
+            passRateDrop: 5,
+            maxLatencyIncreaseMs: 200,
+            maxCostIncreaseUsd: 0.05,
+        },
+        goldenEval: {
+            score: 100,
+            passRate: 100,
+            totalCases: 3,
+            passedCases: 3,
+        },
+        confidenceTests: {
+            passed: testsPassed,
+            total: testTotal,
+        },
+        productMetrics: {},
+    };
+    fs.writeFileSync(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`);
+    ok("Created evals/baseline.json");
+    return true;
+}
+function getHeadSha(cwd) {
+    try {
+        const result = (0, node_child_process_1.spawnSync)("git", ["rev-parse", "--short", "HEAD"], {
+            cwd,
+            stdio: "pipe",
+        });
+        return result.stdout?.toString().trim() || "0000000";
+    }
+    catch {
+        return "0000000";
+    }
 }
+// ── 2. Install GitHub Actions workflow ──
+function installWorkflow(cwd, project) {
+    const workflowDir = path.join(cwd, ".github", "workflows");
+    const workflowPath = path.join(workflowDir, "evalai-gate.yml");
+    if (fs.existsSync(workflowPath)) {
+        skip(".github/workflows/evalai-gate.yml already exists");
+        return true;
+    }
+    if (!fs.existsSync(workflowDir)) {
+        fs.mkdirSync(workflowDir, { recursive: true });
+    }
+    const installCmd = project.pm === "pnpm"
+        ? "pnpm install --frozen-lockfile"
+        : project.pm === "yarn"
+            ? "yarn install --frozen-lockfile"
+            : "npm ci";
+    const setupSteps = project.pm === "pnpm"
+        ? `      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: pnpm
+      - run: ${installCmd}`
+        : `      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: ${project.pm}
+      - run: ${installCmd}`;
+    const workflow = `# EvalAI Regression Gate
+# Auto-generated by: npx evalai init
+# Blocks PRs that regress test health.
+name: EvalAI Gate
+
+on:
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: evalai-\${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  regression-gate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+${setupSteps}
+      - name: EvalAI Regression Gate
+        run: npx -y @pauly4010/evalai-sdk@^1 gate --format github
+
+      - name: Upload report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: regression-report
+          path: evals/regression-report.json
+          if-no-files-found: ignore
 `;
-function runInit(cwd = process.cwd()) {
+    fs.writeFileSync(workflowPath, workflow);
+    ok("Created .github/workflows/evalai-gate.yml");
+    return true;
+}
+// ── 3. Create evalai.config.json ──
+function createConfig(cwd) {
     const configPath = path.join(cwd, "evalai.config.json");
     if (fs.existsSync(configPath)) {
-        console.log(`evalai.config.json already exists at ${path.resolve(configPath)}`);
+        skip("evalai.config.json already exists");
+        return true;
+    }
+    const config = {
+        evaluationId: "",
+        gate: {
+            baseline: "evals/baseline.json",
+            report: "evals/regression-report.json",
+        },
+    };
+    fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`);
+    ok("Created evalai.config.json");
+    return true;
+}
+// ── Main ──
+function runInit(cwd = process.cwd()) {
+    console.log("");
+    console.log("  evalai init — setting up regression gate\n");
+    // Detect
+    const project = detectProject(cwd);
+    if (!project) {
+        console.error("  ✖ No package.json found. Run this from a Node.js project root.");
         return false;
     }
-    fs.writeFileSync(configPath, CONFIG_CONTENT, "utf-8");
-    const resolvedPath = path.resolve(configPath);
-    console.log(`Wrote evalai.config.json at ${resolvedPath}`);
+    ok(`Detected ${project.pm} project: ${project.name}`);
+    if (!project.hasTestScript) {
+        console.log(`  ⚠ No test script found in package.json`);
+        console.log(`    The gate will still work — add a "test" script later for full coverage.\n`);
+    }
+    // Scaffold
+    createBaseline(cwd, project);
+    installWorkflow(cwd, project);
+    createConfig(cwd);
+    // Next steps
+    console.log("");
+    console.log("  Done! Next:");
+    console.log("");
+    console.log("    git add evals/ .github/workflows/evalai-gate.yml evalai.config.json");
+    console.log("    git commit -m 'chore: add EvalAI regression gate'");
+    console.log("    git push");
+    console.log("");
+    console.log("  That's it. Open a PR and the gate runs automatically.");
     console.log("");
-    console.log("Next: paste evaluationId into evalai.config.json, then run npx -y @pauly4010/evalai-sdk@^1 check --format github --onFail import");
+    console.log("  Commands:");
+    console.log("    npx evalai gate              Run gate locally");
+    console.log("    npx evalai gate --format json Machine-readable output");
+    console.log("    npx evalai baseline update    Update baseline after intentional changes");
     console.log("");
-    console.log("GitHub Actions snippet (add to your workflow):");
-    console.log("  - name: EvalAI gate");
-    console.log("    env:");
-    console.log("      EVALAI_API_KEY: ${{ secrets.EVALAI_API_KEY }}");
-    console.log("    run: npx -y @pauly4010/evalai-sdk@^1 check --format github --onFail import");
+    console.log("  To remove: delete evals/, evalai.config.json, and .github/workflows/evalai-gate.yml");
     console.log("");
-    console.log("To uninstall: delete evalai.config.json.");
     return true;
 }

package/dist/cli/regression-gate.d.ts

@@ -1,8 +1,12 @@
 /**
  * evalai gate — Run the regression gate
  *
- * Delegates to the project's eval:regression-gate npm script.
- * Supports --format json to output the regression-report.json contents.
+ * Two modes:
+ *   1. Project mode: delegates to eval:regression-gate npm script (full gate)
+ *   2. Built-in mode: runs `npm test`, compares against evals/baseline.json
+ *
+ * Built-in mode activates when no eval:regression-gate script is defined,
+ * making `npx evalai gate` work for any project after `npx evalai init`.
  */
 export interface GateArgs {
     format: "human" | "json" | "github";

package/dist/cli/regression-gate.js

@@ -2,8 +2,12 @@
 /**
  * evalai gate — Run the regression gate
  *
- * Delegates to the project's eval:regression-gate npm script.
- * Supports --format json to output the regression-report.json contents.
+ * Two modes:
+ *   1. Project mode: delegates to eval:regression-gate npm script (full gate)
+ *   2. Built-in mode: runs `npm test`, compares against evals/baseline.json
+ *
+ * Built-in mode activates when no eval:regression-gate script is defined,
+ * making `npx evalai gate` work for any project after `npx evalai init`.
  */
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
@@ -45,6 +49,7 @@ const node_child_process_1 = require("node:child_process");
 const fs = __importStar(require("node:fs"));
 const path = __importStar(require("node:path"));
 const REPORT_REL = "evals/regression-report.json";
+const BASELINE_REL = "evals/baseline.json";
 /** Detect the package manager used in the project */
 function detectPackageManager(cwd) {
     if (fs.existsSync(path.join(cwd, "pnpm-lock.yaml")))
@@ -66,10 +71,206 @@ function parseGateArgs(argv) {
     }
     return args;
 }
+function detectRunner(cwd) {
+    const pkgPath = path.join(cwd, "package.json");
+    try {
+        const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
+        const testCmd = pkg.scripts?.test ?? "";
+        if (testCmd.includes("vitest"))
+            return "vitest";
+        if (testCmd.includes("jest"))
+            return "jest";
+        if (testCmd.includes("mocha"))
+            return "mocha";
+        if (testCmd.includes("node --test"))
+            return "node:test";
+        if (testCmd.includes("ava"))
+            return "ava";
+        if (testCmd.includes("tap"))
+            return "tap";
+    }
+    catch {
+        // ignore
+    }
+    return "unknown";
+}
+function runBuiltinGate(cwd) {
+    const t0 = Date.now();
+    const baselinePath = path.join(cwd, BASELINE_REL);
+    const now = new Date().toISOString();
+    const pm = detectPackageManager(cwd);
+    const command = `${pm} test`;
+    const runner = detectRunner(cwd);
+    // Load baseline
+    if (!fs.existsSync(baselinePath)) {
+        return {
+            schemaVersion: 1,
+            timestamp: now,
+            exitCode: 2,
+            category: "infra_error",
+            passed: false,
+            failures: ["Baseline file not found. Run: npx evalai init"],
+            deltas: [],
+            baseline: null,
+            durationMs: Date.now() - t0,
+            command,
+            runner,
+        };
+    }
+    let baselineData;
+    try {
+        baselineData = JSON.parse(fs.readFileSync(baselinePath, "utf-8"));
+    }
+    catch {
+        return {
+            schemaVersion: 1,
+            timestamp: now,
+            exitCode: 2,
+            category: "infra_error",
+            passed: false,
+            failures: ["Failed to parse evals/baseline.json"],
+            deltas: [],
+            baseline: null,
+            durationMs: Date.now() - t0,
+            command,
+            runner,
+        };
+    }
+    const baselineMeta = baselineData.updatedAt
+        ? { updatedAt: baselineData.updatedAt, updatedBy: baselineData.updatedBy ?? "unknown" }
+        : null;
+    // Run tests
+    const isWin = process.platform === "win32";
+    const result = (0, node_child_process_1.spawnSync)(pm, ["test"], {
+        cwd,
+        stdio: "pipe",
+        shell: isWin,
+        timeout: 300000,
+    });
+    const testsPassed = result.status === 0;
+    const output = (result.stdout?.toString() ?? "") + (result.stderr?.toString() ?? "");
+    // Try to extract test count
+    let testCount = 0;
+    const countMatch = output.match(/(\d+)\s+(?:tests?|specs?)\s+(?:passed|completed)/i) ??
+        output.match(/Tests:\s+(\d+)\s+passed/i) ??
+        output.match(/(\d+)\s+passing/i) ??
+        output.match(/Test Files\s+\d+\s+passed.*\n\s+Tests\s+(\d+)\s+passed/i);
+    if (countMatch)
+        testCount = parseInt(countMatch[1], 10);
+    // Compare against baseline
+    const baselinePassed = baselineData.confidenceTests?.passed ?? true;
+    const baselineTotal = baselineData.confidenceTests?.total ?? 0;
+    const failures = [];
+    const deltas = [];
+    // Delta: tests passing
+    deltas.push({
+        metric: "tests_passing",
+        baseline: baselinePassed,
+        current: testsPassed,
+        delta: testsPassed === baselinePassed ? "0" : testsPassed ? "+1" : "-1",
+        status: testsPassed ? "pass" : "fail",
+    });
+    if (!testsPassed && baselinePassed) {
+        failures.push("Tests were passing in baseline but are now failing");
+    }
+    // Delta: test count (only if we captured counts)
+    if (testCount > 0 || baselineTotal > 0) {
+        const countDelta = testCount - baselineTotal;
+        deltas.push({
+            metric: "test_count",
+            baseline: baselineTotal,
+            current: testCount,
+            delta: countDelta >= 0 ? `+${countDelta}` : `${countDelta}`,
+            status: testCount >= baselineTotal ? "pass" : "fail",
+        });
+        if (testCount < baselineTotal) {
+            failures.push(`Test count dropped from ${baselineTotal} to ${testCount} (${countDelta})`);
+        }
+    }
+    const hasRegression = failures.length > 0;
+    return {
+        schemaVersion: 1,
+        timestamp: now,
+        exitCode: hasRegression ? 1 : 0,
+        category: hasRegression ? "regression" : "pass",
+        passed: !hasRegression,
+        failures,
+        deltas,
+        baseline: baselineMeta,
+        durationMs: Date.now() - t0,
+        command,
+        runner,
+    };
+}
+// ── Format helpers ──
+function formatHuman(report) {
+    const icon = report.passed ? "✅" : "❌";
+    console.log(`\n${icon} EvalAI Gate: ${report.category.toUpperCase()}\n`);
+    if (report.deltas.length > 0) {
+        const pad = (s, n) => s.padEnd(n);
+        console.log(`  ${pad("Metric", 16)} ${pad("Baseline", 10)} ${pad("Current", 10)} ${pad("Delta", 8)} Status`);
+        console.log(`  ${"-".repeat(16)} ${"-".repeat(10)} ${"-".repeat(10)} ${"-".repeat(8)} ------`);
+        for (const d of report.deltas) {
+            const si = d.status === "pass" ? "✔" : "✖";
+            console.log(`  ${pad(d.metric, 16)} ${pad(String(d.baseline), 10)} ${pad(String(d.current), 10)} ${pad(d.delta, 8)} ${si}`);
+        }
+    }
+    if (report.failures.length > 0) {
+        console.log("\n  Failures:");
+        for (const f of report.failures) {
+            console.log(`    • ${f}`);
+        }
+    }
+    console.log("");
+}
+function formatGithub(report) {
+    const icon = report.passed ? "✅" : "❌";
+    const lines = [
+        `## ${icon} EvalAI Gate: ${report.category}`,
+        "",
+        "| Metric | Baseline | Current | Delta | Status |",
+        "|--------|----------|---------|-------|--------|",
+    ];
+    for (const d of report.deltas) {
+        const si = d.status === "pass" ? "✅" : "❌";
+        lines.push(`| ${d.metric} | ${d.baseline} | ${d.current} | ${d.delta} | ${si} |`);
+    }
+    if (report.failures.length > 0) {
+        lines.push("", "### Failures", "");
+        for (const f of report.failures) {
+            lines.push(`- ${f}`);
+        }
+    }
+    lines.push("", `Schema version: ${report.schemaVersion}`);
+    const md = lines.join("\n");
+    // Write to $GITHUB_STEP_SUMMARY if available
+    const summaryPath = process.env.GITHUB_STEP_SUMMARY;
+    if (summaryPath) {
+        try {
+            fs.appendFileSync(summaryPath, `${md}\n`);
+        }
+        catch {
+            // ignore if not writable
+        }
+    }
+    console.log(md);
+}
+function formatReport(report, args) {
+    if (args.format === "json") {
+        process.stdout.write(JSON.stringify(report, null, 2));
+    }
+    else if (args.format === "github") {
+        formatGithub(report);
+    }
+    else {
+        formatHuman(report);
+    }
+}
+// ── Main ──
 function runGate(argv) {
     const cwd = process.cwd();
     const args = parseGateArgs(argv);
-    // Check if eval:regression-gate script exists
+    // Check for package.json
     const pkgPath = path.join(cwd, "package.json");
     if (!fs.existsSync(pkgPath)) {
         console.error("❌ No package.json found. Run this from your project root.");
@@ -83,68 +284,52 @@ function runGate(argv) {
         console.error("❌ Failed to parse package.json");
         return 1;
     }
-    if (!pkg.scripts?.["eval:regression-gate"]) {
-        console.error("❌ Missing 'eval:regression-gate' script in package.json.");
-        console.error('   Add it:  "eval:regression-gate": "npx tsx scripts/regression-gate.ts"');
-        return 1;
-    }
-    const pm = detectPackageManager(cwd);
-    const isWin = process.platform === "win32";
-    // For json format, suppress human output and print report JSON
-    const stdio = args.format === "json" ? "pipe" : "inherit";
-    const result = (0, node_child_process_1.spawnSync)(pm, ["run", "eval:regression-gate"], {
-        cwd,
-        stdio: stdio,
-        shell: isWin,
-    });
-    const exitCode = result.status ?? 1;
-    if (args.format === "json") {
-        // Output the regression report as JSON
-        const reportPath = path.join(cwd, REPORT_REL);
-        if (fs.existsSync(reportPath)) {
-            const report = fs.readFileSync(reportPath, "utf-8");
-            process.stdout.write(report);
-        }
-        else {
-            console.error(JSON.stringify({ error: "regression-report.json not found", exitCode }));
+    // ── Project mode: delegate to eval:regression-gate if it exists ──
+    if (pkg.scripts?.["eval:regression-gate"]) {
+        const pm = detectPackageManager(cwd);
+        const isWin = process.platform === "win32";
+        const stdio = args.format === "json" ? "pipe" : "inherit";
+        const result = (0, node_child_process_1.spawnSync)(pm, ["run", "eval:regression-gate"], {
+            cwd,
+            stdio: stdio,
+            shell: isWin,
+        });
+        const exitCode = result.status ?? 1;
+        // Post-process report for json/github formats
+        if (args.format === "json") {
+            const reportPath = path.join(cwd, REPORT_REL);
+            if (fs.existsSync(reportPath)) {
+                process.stdout.write(fs.readFileSync(reportPath, "utf-8"));
+            }
+            else {
+                console.error(JSON.stringify({ error: "regression-report.json not found", exitCode }));
+            }
         }
-    }
-    else if (args.format === "github") {
-        // Output GitHub Step Summary markdown
-        const reportPath = path.join(cwd, REPORT_REL);
-        if (fs.existsSync(reportPath)) {
-            try {
-                const report = JSON.parse(fs.readFileSync(reportPath, "utf-8"));
-                const icon = report.passed ? "✅" : "❌";
-                const lines = [
-                    `## ${icon} Regression Gate: ${report.category}`,
-                    "",
-                    "| Metric | Baseline | Current | Delta | Status |",
-                    "|--------|----------|---------|-------|--------|",
-                ];
-                for (const d of report.deltas ?? []) {
-                    const statusIcon = d.status === "pass" ? "✅" : "❌";
-                    lines.push(`| ${d.metric} | ${d.baseline} | ${d.current} | ${d.delta} | ${statusIcon} |`);
-                }
-                if (report.failures?.length > 0) {
-                    lines.push("", "### Failures", "");
-                    for (const f of report.failures) {
-                        lines.push(`- ${f}`);
-                    }
+        else if (args.format === "github") {
+            const reportPath = path.join(cwd, REPORT_REL);
+            if (fs.existsSync(reportPath)) {
+                try {
+                    const report = JSON.parse(fs.readFileSync(reportPath, "utf-8"));
+                    formatGithub(report);
                 }
-                lines.push("", `Schema version: ${report.schemaVersion ?? "unknown"}`);
-                const md = lines.join("\n");
-                // Write to $GITHUB_STEP_SUMMARY if available
-                const summaryPath = process.env.GITHUB_STEP_SUMMARY;
-                if (summaryPath) {
-                    fs.appendFileSync(summaryPath, `${md}\n`);
+                catch {
+                    // human output already printed
                 }
-                console.log(md);
-            }
-            catch {
-                // Fall through — human output already printed
             }
         }
+        return exitCode;
+    }
+    // ── Built-in mode: run tests + compare against baseline ──
+    if (args.format === "human") {
+        console.log("\n  Running EvalAI regression gate (built-in mode)...\n");
+    }
+    const report = runBuiltinGate(cwd);
+    // Write report artifact
+    const evalsDir = path.join(cwd, "evals");
+    if (!fs.existsSync(evalsDir)) {
+        fs.mkdirSync(evalsDir, { recursive: true });
     }
-    return exitCode;
+    fs.writeFileSync(path.join(cwd, REPORT_REL), `${JSON.stringify(report, null, 2)}\n`);
+    formatReport(report, args);
+    return report.exitCode;
 }

package/dist/cli/upgrade.d.ts

@@ -0,0 +1,15 @@
+/**
+ * evalai upgrade --full — Upgrade from Tier 1 (built-in gate) to Tier 2 (full gate)
+ *
+ * What it does:
+ *   1. Adds full regression gate script (scripts/regression-gate.ts)
+ *   2. Adds baseline governance workflow (.github/workflows/baseline-governance.yml)
+ *   3. Updates package.json with eval:regression-gate + eval:baseline-update scripts
+ *   4. Updates .github/workflows/evalai-gate.yml to use project mode
+ *   5. Prints next steps
+ */
+export interface UpgradeArgs {
+    full: boolean;
+}
+export declare function parseUpgradeArgs(argv: string[]): UpgradeArgs;
+export declare function runUpgrade(argv: string[]): number;