@pauly4010/evalai-sdk 1.5.8 → 1.6.0

package/CHANGELOG.md

@@ -5,6 +5,34 @@ All notable changes to the @pauly4010/evalai-sdk package will be documented in t
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.6.0] - 2026-02-24
+
+### ✨ Added
+
+#### CLI — Regression Gate & Baseline Management
+
+- **`evalai baseline init`** — Create a starter `evals/baseline.json` with sample values and provenance metadata
+- **`evalai baseline update`** — Run confidence tests, golden eval, and latency benchmark, then update baseline with real scores
+- **`evalai gate`** — Run the local regression gate with proper exit code taxonomy (0=pass, 1=regression, 2=infra_error, 3=confidence_failed, 4=confidence_missing)
+- **`evalai gate --format json`** — Output `evals/regression-report.json` as machine-readable JSON to stdout
+- **`evalai gate --format github`** — Output GitHub Step Summary markdown with delta table
+
+#### SDK Exports — Regression Gate Constants & Types
+
+- **`GATE_EXIT`** — Exit code constants (`PASS`, `REGRESSION`, `INFRA_ERROR`, `CONFIDENCE_FAILED`, `CONFIDENCE_MISSING`)
+- **`GATE_CATEGORY`** — Report category constants (`pass`, `regression`, `infra_error`)
+- **`REPORT_SCHEMA_VERSION`** — Current schema version for `regression-report.json`
+- **`ARTIFACTS`** — Well-known artifact paths (`BASELINE`, `REGRESSION_REPORT`, `CONFIDENCE_SUMMARY`, `LATENCY_BENCHMARK`)
+- **Types**: `RegressionReport`, `RegressionDelta`, `Baseline`, `BaselineTolerance`, `GateExitCode`, `GateCategory`
+- **Subpath export**: `@pauly4010/evalai-sdk/regression` for tree-shakeable imports
+
+### 🔧 Changed
+
+- CLI help text updated to include `baseline` and `gate` commands
+- SDK becomes the public contract for regression gate — scripts are implementation detail
+
+---
+
 ## [1.5.8] - 2026-02-22
 
 ### 🐛 Fixed

package/dist/cli/baseline.d.ts

@@ -0,0 +1,10 @@
+/**
+ * evalai baseline — Baseline management commands
+ *
+ * Subcommands:
+ *   evalai baseline init    — Create a starter evals/baseline.json
+ *   evalai baseline update  — Run tests + update baseline with real scores
+ */
+export declare function runBaselineInit(cwd: string): number;
+export declare function runBaselineUpdate(cwd: string): number;
+export declare function runBaseline(argv: string[]): number;

package/dist/cli/baseline.js

@@ -0,0 +1,172 @@
+"use strict";
+/**
+ * evalai baseline — Baseline management commands
+ *
+ * Subcommands:
+ *   evalai baseline init    — Create a starter evals/baseline.json
+ *   evalai baseline update  — Run tests + update baseline with real scores
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runBaselineInit = runBaselineInit;
+exports.runBaselineUpdate = runBaselineUpdate;
+exports.runBaseline = runBaseline;
+const node_child_process_1 = require("node:child_process");
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const BASELINE_REL = "evals/baseline.json";
+/** Detect the package manager used in the project */
+function detectPackageManager(cwd) {
+    if (fs.existsSync(path.join(cwd, "pnpm-lock.yaml")))
+        return "pnpm";
+    if (fs.existsSync(path.join(cwd, "yarn.lock")))
+        return "yarn";
+    return "npm";
+}
+/** Run an npm script via the detected package manager */
+function runScript(cwd, scriptName) {
+    const pm = detectPackageManager(cwd);
+    const isWin = process.platform === "win32";
+    const result = (0, node_child_process_1.spawnSync)(pm, ["run", scriptName], {
+        cwd,
+        stdio: "inherit",
+        shell: isWin,
+    });
+    return result.status ?? 1;
+}
+function runBaselineInit(cwd) {
+    const baselinePath = path.join(cwd, BASELINE_REL);
+    if (fs.existsSync(baselinePath)) {
+        console.log(`⚠ ${BASELINE_REL} already exists. Delete it first or use 'evalai baseline update'.`);
+        return 1;
+    }
+    // Ensure evals/ directory exists
+    const evalsDir = path.join(cwd, "evals");
+    if (!fs.existsSync(evalsDir)) {
+        fs.mkdirSync(evalsDir, { recursive: true });
+    }
+    const user = process.env.USER || process.env.USERNAME || "unknown";
+    const now = new Date().toISOString();
+    const baseline = {
+        schemaVersion: 1,
+        description: "Regression gate baseline — created by evalai baseline init",
+        generatedAt: now,
+        generatedBy: user,
+        commitSha: "0000000",
+        updatedAt: now,
+        updatedBy: user,
+        tolerance: {
+            scoreDrop: 5,
+            passRateDrop: 5,
+            maxLatencyIncreaseMs: 200,
+            maxCostIncreaseUsd: 0.05,
+        },
+        goldenEval: {
+            score: 100,
+            passRate: 100,
+            totalCases: 3,
+            passedCases: 3,
+        },
+        qualityScore: {
+            overall: 90,
+            grade: "A",
+            accuracy: 85,
+            safety: 100,
+            latency: 90,
+            cost: 90,
+            consistency: 90,
+        },
+        confidenceTests: {
+            unitPassed: true,
+            unitTotal: 0,
+            dbPassed: true,
+            dbTotal: 0,
+        },
+        productMetrics: {},
+    };
+    fs.writeFileSync(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`);
+    console.log(`✅ Created ${BASELINE_REL} with sample values\n`);
+    console.log("Next steps:");
+    console.log(`  1. Commit ${BASELINE_REL} to your repo`);
+    console.log("  2. Run 'evalai baseline update' to populate with real scores");
+    console.log("  3. Run 'evalai gate' to verify the regression gate\n");
+    return 0;
+}
+// ── baseline update ──
+function runBaselineUpdate(cwd) {
+    // Check if eval:baseline-update script exists in package.json
+    const pkgPath = path.join(cwd, "package.json");
+    if (!fs.existsSync(pkgPath)) {
+        console.error("❌ No package.json found. Run this from your project root.");
+        return 1;
+    }
+    let pkg;
+    try {
+        pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
+    }
+    catch {
+        console.error("❌ Failed to parse package.json");
+        return 1;
+    }
+    if (!pkg.scripts?.["eval:baseline-update"]) {
+        console.error("❌ Missing 'eval:baseline-update' script in package.json.");
+        console.error("   Add it:  \"eval:baseline-update\": \"npx tsx scripts/regression-gate.ts --update-baseline\"");
+        return 1;
+    }
+    console.log("📊 Running baseline update...\n");
+    return runScript(cwd, "eval:baseline-update");
+}
+// ── baseline router ──
+function runBaseline(argv) {
+    const sub = argv[0];
+    const cwd = process.cwd();
+    if (sub === "init") {
+        return runBaselineInit(cwd);
+    }
+    if (sub === "update") {
+        return runBaselineUpdate(cwd);
+    }
+    console.log(`evalai baseline — Manage regression gate baselines
+
+Usage:
+  evalai baseline init     Create starter ${BASELINE_REL}
+  evalai baseline update   Run tests and update baseline with real scores
+
+Examples:
+  evalai baseline init
+  evalai baseline update
+`);
+    return sub === "--help" || sub === "-h" ? 0 : 1;
+}

package/dist/cli/index.js

@@ -8,9 +8,11 @@
  *   evalai check  — CI/CD evaluation gate (see evalai check --help)
  */
 Object.defineProperty(exports, "__esModule", { value: true });
+const baseline_1 = require("./baseline");
 const check_1 = require("./check");
 const doctor_1 = require("./doctor");
 const init_1 = require("./init");
+const regression_gate_1 = require("./regression-gate");
 const share_1 = require("./share");
 const argv = process.argv.slice(2);
 const subcommand = argv[0];
@@ -19,6 +21,14 @@ if (subcommand === "init") {
     const ok = (0, init_1.runInit)(cwd);
     process.exit(ok ? 0 : 1);
 }
+else if (subcommand === "baseline") {
+    const code = (0, baseline_1.runBaseline)(argv.slice(1));
+    process.exit(code);
+}
+else if (subcommand === "gate") {
+    const code = (0, regression_gate_1.runGate)(argv.slice(1));
+    process.exit(code);
+}
 else if (subcommand === "doctor") {
     (0, doctor_1.runDoctor)(argv.slice(1))
         .then((code) => process.exit(code))
@@ -57,10 +67,16 @@ else {
     console.log(`EvalAI CLI
 
 Usage:
-  evalai init              Create evalai.config.json
-  evalai doctor [options]  Verify CI/CD setup (same endpoint as check)
-  evalai check [options]   CI/CD evaluation gate
-  evalai share [options]   Create share link for a run
+  evalai init                Create evalai.config.json
+  evalai baseline init       Create starter evals/baseline.json
+  evalai baseline update     Run tests and update baseline with real scores
+  evalai gate [options]      Run regression gate (local test-based)
+  evalai doctor [options]    Verify CI/CD setup (same endpoint as check)
+  evalai check [options]     CI/CD evaluation gate (API-based)
+  evalai share [options]     Create share link for a run
+
+Options for gate:
+  --format <fmt>      Output format: human (default), json, github
 
 Options for check:
   --evaluationId <id>  Evaluation to gate on (or from config)

package/dist/cli/regression-gate.d.ts

@@ -0,0 +1,11 @@
+/**
+ * evalai gate — Run the regression gate
+ *
+ * Delegates to the project's eval:regression-gate npm script.
+ * Supports --format json to output the regression-report.json contents.
+ */
+export interface GateArgs {
+    format: "human" | "json" | "github";
+}
+export declare function parseGateArgs(argv: string[]): GateArgs;
+export declare function runGate(argv: string[]): number;

package/dist/cli/regression-gate.js

@@ -0,0 +1,150 @@
+"use strict";
+/**
+ * evalai gate — Run the regression gate
+ *
+ * Delegates to the project's eval:regression-gate npm script.
+ * Supports --format json to output the regression-report.json contents.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseGateArgs = parseGateArgs;
+exports.runGate = runGate;
+const node_child_process_1 = require("node:child_process");
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const REPORT_REL = "evals/regression-report.json";
+/** Detect the package manager used in the project */
+function detectPackageManager(cwd) {
+    if (fs.existsSync(path.join(cwd, "pnpm-lock.yaml")))
+        return "pnpm";
+    if (fs.existsSync(path.join(cwd, "yarn.lock")))
+        return "yarn";
+    return "npm";
+}
+function parseGateArgs(argv) {
+    const args = { format: "human" };
+    for (let i = 0; i < argv.length; i++) {
+        if (argv[i] === "--format" && argv[i + 1]) {
+            const fmt = argv[i + 1];
+            if (fmt === "json" || fmt === "github" || fmt === "human") {
+                args.format = fmt;
+            }
+            i++;
+        }
+    }
+    return args;
+}
+function runGate(argv) {
+    const cwd = process.cwd();
+    const args = parseGateArgs(argv);
+    // Check if eval:regression-gate script exists
+    const pkgPath = path.join(cwd, "package.json");
+    if (!fs.existsSync(pkgPath)) {
+        console.error("❌ No package.json found. Run this from your project root.");
+        return 1;
+    }
+    let pkg;
+    try {
+        pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
+    }
+    catch {
+        console.error("❌ Failed to parse package.json");
+        return 1;
+    }
+    if (!pkg.scripts?.["eval:regression-gate"]) {
+        console.error("❌ Missing 'eval:regression-gate' script in package.json.");
+        console.error('   Add it:  "eval:regression-gate": "npx tsx scripts/regression-gate.ts"');
+        return 1;
+    }
+    const pm = detectPackageManager(cwd);
+    const isWin = process.platform === "win32";
+    // For json format, suppress human output and print report JSON
+    const stdio = args.format === "json" ? "pipe" : "inherit";
+    const result = (0, node_child_process_1.spawnSync)(pm, ["run", "eval:regression-gate"], {
+        cwd,
+        stdio: stdio,
+        shell: isWin,
+    });
+    const exitCode = result.status ?? 1;
+    if (args.format === "json") {
+        // Output the regression report as JSON
+        const reportPath = path.join(cwd, REPORT_REL);
+        if (fs.existsSync(reportPath)) {
+            const report = fs.readFileSync(reportPath, "utf-8");
+            process.stdout.write(report);
+        }
+        else {
+            console.error(JSON.stringify({ error: "regression-report.json not found", exitCode }));
+        }
+    }
+    else if (args.format === "github") {
+        // Output GitHub Step Summary markdown
+        const reportPath = path.join(cwd, REPORT_REL);
+        if (fs.existsSync(reportPath)) {
+            try {
+                const report = JSON.parse(fs.readFileSync(reportPath, "utf-8"));
+                const icon = report.passed ? "✅" : "❌";
+                const lines = [
+                    `## ${icon} Regression Gate: ${report.category}`,
+                    "",
+                    "| Metric | Baseline | Current | Delta | Status |",
+                    "|--------|----------|---------|-------|--------|",
+                ];
+                for (const d of report.deltas ?? []) {
+                    const statusIcon = d.status === "pass" ? "✅" : "❌";
+                    lines.push(`| ${d.metric} | ${d.baseline} | ${d.current} | ${d.delta} | ${statusIcon} |`);
+                }
+                if (report.failures?.length > 0) {
+                    lines.push("", "### Failures", "");
+                    for (const f of report.failures) {
+                        lines.push(`- ${f}`);
+                    }
+                }
+                lines.push("", `Schema version: ${report.schemaVersion ?? "unknown"}`);
+                const md = lines.join("\n");
+                // Write to $GITHUB_STEP_SUMMARY if available
+                const summaryPath = process.env.GITHUB_STEP_SUMMARY;
+                if (summaryPath) {
+                    fs.appendFileSync(summaryPath, `${md}\n`);
+                }
+                console.log(md);
+            }
+            catch {
+                // Fall through — human output already printed
+            }
+        }
+    }
+    return exitCode;
+}

package/dist/client.request.test.d.ts

@@ -1 +1 @@
-export {};
+export {};

package/dist/client.request.test.js

@@ -1,157 +1,157 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-const vitest_1 = require("vitest");
-const client_1 = require("./client");
-const errorsModule = __importStar(require("./errors"));
-vitest_1.vi.mock("./cache", () => {
-    const cacheTracker = { invalidatedPatterns: [] };
-    const shouldCache = vitest_1.vi.fn().mockReturnValue(true);
-    const getTTL = vitest_1.vi.fn().mockReturnValue(1000);
-    const makeKey = (method, url, params) => `${method}:${url}:${JSON.stringify(params ?? null)}`;
-    return {
-        __esModule: true,
-        shouldCache,
-        getTTL,
-        cacheTracker,
-        RequestCache: class RequestCache {
-            constructor() {
-                this.store = new Map();
-            }
-            get(method, url, params) {
-                const key = makeKey(method, url, params);
-                return this.store.get(key) ?? null;
-            }
-            set(method, url, data, _ttl, params) {
-                const key = makeKey(method, url, params);
-                this.store.set(key, data);
-            }
-            invalidatePattern(pattern) {
-                cacheTracker.invalidatedPatterns.push(pattern);
-            }
-            invalidate(_method, _url, _params) {
-                // no-op for tests
-            }
-            clear() {
-                this.store.clear();
-            }
-        },
-    };
-});
-const cache_1 = require("./cache");
-(0, vitest_1.describe)("AIEvalClient.request", () => {
-    (0, vitest_1.beforeEach)(() => {
-        process.env.EVALAI_API_KEY = "test";
-        cache_1.shouldCache.mockReset().mockReturnValue(true);
-        cache_1.getTTL.mockReset().mockReturnValue(1000);
-        cache_1.cacheTracker.invalidatedPatterns.length = 0;
-    });
-    (0, vitest_1.it)("caches GET responses and reuses data without re-fetching", async () => {
-        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost", timeout: 1000 });
-        const payload = { items: [1, 2, 3] };
-        const fetchMock = vitest_1.vi.fn().mockResolvedValue({
-            ok: true,
-            status: 200,
-            json: async () => payload,
-        });
-        globalThis.fetch = fetchMock;
-        const first = await client.request("/api/traces", { method: "GET" });
-        const second = await client.request("/api/traces", { method: "GET" });
-        (0, vitest_1.expect)(first).toEqual(payload);
-        (0, vitest_1.expect)(second).toEqual(payload);
-        (0, vitest_1.expect)(fetchMock).toHaveBeenCalledTimes(1);
-    });
-    (0, vitest_1.it)("propagates non-ok responses as SDK errors", async () => {
-        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost" });
-        const fetchMock = vitest_1.vi.fn().mockResolvedValue({
-            ok: false,
-            status: 429,
-            json: async () => ({ error: { code: "RATE_LIMIT_EXCEEDED" } }),
-        });
-        globalThis.fetch = fetchMock;
-        const createErrorSpy = vitest_1.vi
-            .spyOn(errorsModule, "createErrorFromResponse")
-            .mockReturnValue(new errorsModule.EvalAIError("rate limited", "RATE_LIMIT_EXCEEDED", 429));
-        await (0, vitest_1.expect)(client.request("/api/fail", { method: "GET" })).rejects.toHaveProperty("code", "RATE_LIMIT_EXCEEDED");
-        createErrorSpy.mockRestore();
-    });
-    (0, vitest_1.it)("retries on retryable SDK errors and eventually succeeds", async () => {
-        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost", timeout: 1000 });
-        vitest_1.vi.spyOn(client, "calculateBackoff").mockReturnValue(0);
-        const failureResponse = {
-            ok: false,
-            status: 429,
-            json: async () => ({ error: { code: "RATE_LIMIT_EXCEEDED" } }),
-        };
-        const successResponse = {
-            ok: true,
-            status: 200,
-            json: async () => ({ ok: true }),
-        };
-        const createErrorSpy = vitest_1.vi
-            .spyOn(errorsModule, "createErrorFromResponse")
-            .mockReturnValue(new errorsModule.EvalAIError("rate limited", "RATE_LIMIT_EXCEEDED", 429));
-        const fetchMock = vitest_1.vi
-            .fn()
-            .mockResolvedValueOnce(failureResponse)
-            .mockResolvedValueOnce(successResponse);
-        globalThis.fetch = fetchMock;
-        const result = await client.request("/api/retry", { method: "GET" });
-        (0, vitest_1.expect)(result).toEqual({ ok: true });
-        (0, vitest_1.expect)(fetchMock).toHaveBeenCalledTimes(2);
-        createErrorSpy.mockRestore();
-    });
-    (0, vitest_1.it)("throws a TIMEOUT SDK error when fetch aborts", async () => {
-        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost", timeout: 1000 });
-        const abortError = Object.assign(new Error("aborted"), { name: "AbortError" });
-        const fetchMock = vitest_1.vi.fn().mockRejectedValue(abortError);
-        globalThis.fetch = fetchMock;
-        await (0, vitest_1.expect)(client.request("/api/timeout", { method: "GET" })).rejects.toMatchObject({
-            code: "TIMEOUT",
-        });
-    });
-    (0, vitest_1.it)("invalidates related cache entries for mutation requests", async () => {
-        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost", timeout: 1000 });
-        cache_1.shouldCache.mockReturnValue(false);
-        const fetchMock = vitest_1.vi.fn().mockResolvedValue({
-            ok: true,
-            status: 201,
-            json: async () => ({ result: "ok" }),
-        });
-        globalThis.fetch = fetchMock;
-        await client.request("/api/evaluations", { method: "POST", body: JSON.stringify({}) });
-        (0, vitest_1.expect)(cache_1.cacheTracker.invalidatedPatterns).toContain("evaluations");
-    });
-});
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const client_1 = require("./client");
+const errorsModule = __importStar(require("./errors"));
+vitest_1.vi.mock("./cache", () => {
+    const cacheTracker = { invalidatedPatterns: [] };
+    const shouldCache = vitest_1.vi.fn().mockReturnValue(true);
+    const getTTL = vitest_1.vi.fn().mockReturnValue(1000);
+    const makeKey = (method, url, params) => `${method}:${url}:${JSON.stringify(params ?? null)}`;
+    return {
+        __esModule: true,
+        shouldCache,
+        getTTL,
+        cacheTracker,
+        RequestCache: class RequestCache {
+            constructor() {
+                this.store = new Map();
+            }
+            get(method, url, params) {
+                const key = makeKey(method, url, params);
+                return this.store.get(key) ?? null;
+            }
+            set(method, url, data, _ttl, params) {
+                const key = makeKey(method, url, params);
+                this.store.set(key, data);
+            }
+            invalidatePattern(pattern) {
+                cacheTracker.invalidatedPatterns.push(pattern);
+            }
+            invalidate(_method, _url, _params) {
+                // no-op for tests
+            }
+            clear() {
+                this.store.clear();
+            }
+        },
+    };
+});
+const cache_1 = require("./cache");
+(0, vitest_1.describe)("AIEvalClient.request", () => {
+    (0, vitest_1.beforeEach)(() => {
+        process.env.EVALAI_API_KEY = "test";
+        cache_1.shouldCache.mockReset().mockReturnValue(true);
+        cache_1.getTTL.mockReset().mockReturnValue(1000);
+        cache_1.cacheTracker.invalidatedPatterns.length = 0;
+    });
+    (0, vitest_1.it)("caches GET responses and reuses data without re-fetching", async () => {
+        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost", timeout: 1000 });
+        const payload = { items: [1, 2, 3] };
+        const fetchMock = vitest_1.vi.fn().mockResolvedValue({
+            ok: true,
+            status: 200,
+            json: async () => payload,
+        });
+        globalThis.fetch = fetchMock;
+        const first = await client.request("/api/traces", { method: "GET" });
+        const second = await client.request("/api/traces", { method: "GET" });
+        (0, vitest_1.expect)(first).toEqual(payload);
+        (0, vitest_1.expect)(second).toEqual(payload);
+        (0, vitest_1.expect)(fetchMock).toHaveBeenCalledTimes(1);
+    });
+    (0, vitest_1.it)("propagates non-ok responses as SDK errors", async () => {
+        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost" });
+        const fetchMock = vitest_1.vi.fn().mockResolvedValue({
+            ok: false,
+            status: 429,
+            json: async () => ({ error: { code: "RATE_LIMIT_EXCEEDED" } }),
+        });
+        globalThis.fetch = fetchMock;
+        const createErrorSpy = vitest_1.vi
+            .spyOn(errorsModule, "createErrorFromResponse")
+            .mockReturnValue(new errorsModule.EvalAIError("rate limited", "RATE_LIMIT_EXCEEDED", 429));
+        await (0, vitest_1.expect)(client.request("/api/fail", { method: "GET" })).rejects.toHaveProperty("code", "RATE_LIMIT_EXCEEDED");
+        createErrorSpy.mockRestore();
+    });
+    (0, vitest_1.it)("retries on retryable SDK errors and eventually succeeds", async () => {
+        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost", timeout: 1000 });
+        vitest_1.vi.spyOn(client, "calculateBackoff").mockReturnValue(0);
+        const failureResponse = {
+            ok: false,
+            status: 429,
+            json: async () => ({ error: { code: "RATE_LIMIT_EXCEEDED" } }),
+        };
+        const successResponse = {
+            ok: true,
+            status: 200,
+            json: async () => ({ ok: true }),
+        };
+        const createErrorSpy = vitest_1.vi
+            .spyOn(errorsModule, "createErrorFromResponse")
+            .mockReturnValue(new errorsModule.EvalAIError("rate limited", "RATE_LIMIT_EXCEEDED", 429));
+        const fetchMock = vitest_1.vi
+            .fn()
+            .mockResolvedValueOnce(failureResponse)
+            .mockResolvedValueOnce(successResponse);
+        globalThis.fetch = fetchMock;
+        const result = await client.request("/api/retry", { method: "GET" });
+        (0, vitest_1.expect)(result).toEqual({ ok: true });
+        (0, vitest_1.expect)(fetchMock).toHaveBeenCalledTimes(2);
+        createErrorSpy.mockRestore();
+    });
+    (0, vitest_1.it)("throws a TIMEOUT SDK error when fetch aborts", async () => {
+        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost", timeout: 1000 });
+        const abortError = Object.assign(new Error("aborted"), { name: "AbortError" });
+        const fetchMock = vitest_1.vi.fn().mockRejectedValue(abortError);
+        globalThis.fetch = fetchMock;
+        await (0, vitest_1.expect)(client.request("/api/timeout", { method: "GET" })).rejects.toMatchObject({
+            code: "TIMEOUT",
+        });
+    });
+    (0, vitest_1.it)("invalidates related cache entries for mutation requests", async () => {
+        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost", timeout: 1000 });
+        cache_1.shouldCache.mockReturnValue(false);
+        const fetchMock = vitest_1.vi.fn().mockResolvedValue({
+            ok: true,
+            status: 201,
+            json: async () => ({ result: "ok" }),
+        });
+        globalThis.fetch = fetchMock;
+        await client.request("/api/evaluations", { method: "POST", body: JSON.stringify({}) });
+        (0, vitest_1.expect)(cache_1.cacheTracker.invalidatedPatterns).toContain("evaluations");
+    });
+});

package/dist/index.d.ts

@@ -33,5 +33,6 @@ export { batchProcess, batchRead, RateLimiter, streamEvaluation } from "./stream
 export type { Annotation, AnnotationItem, AnnotationTask, APIKey, APIKeyUsage, APIKeyWithSecret, BatchOptions, ClientConfig as AIEvalConfig, CreateAnnotationItemParams, CreateAnnotationParams, CreateAnnotationTaskParams, CreateAPIKeyParams, CreateLLMJudgeConfigParams, CreateWebhookParams, Evaluation as EvaluationData, ExportOptions, GenericMetadata as AnnotationData, GetLLMJudgeAlignmentParams, GetUsageParams, ImportOptions, ListAnnotationItemsParams, ListAnnotationsParams, ListAnnotationTasksParams, ListAPIKeysParams, ListLLMJudgeConfigsParams, ListLLMJudgeResultsParams, ListWebhookDeliveriesParams, ListWebhooksParams, LLMJudgeAlignment, LLMJudgeConfig, LLMJudgeResult as LLMJudgeData, Organization, RetryConfig, SnapshotData, Span as SpanData, StreamOptions, TestCase, TestResult, Trace as TraceData, TracedResponse, UpdateAPIKeyParams, UpdateWebhookParams, UsageStats, UsageSummary, Webhook, WebhookDelivery, } from "./types";
 export { EvaluationTemplates, type EvaluationTemplateType, type FeatureUsage, type OrganizationLimits, } from "./types";
 export { type AgentHandoff, type AgentSpanContext, type CostCategory, type CostRecord, createWorkflowTracer, type DecisionAlternative, type DecisionType, type HandoffType, type LLMProvider, type RecordCostParams, type RecordDecisionParams, traceAutoGen, traceCrewAI, traceLangChainAgent, traceWorkflowStep, type WorkflowContext, type WorkflowDefinition, type WorkflowEdge, type WorkflowNode, type WorkflowStatus, WorkflowTracer, type WorkflowTracerOptions, } from "./workflows";
+export { ARTIFACTS, type Baseline, type BaselineTolerance, GATE_CATEGORY, GATE_EXIT, type GateCategory, type GateExitCode, type RegressionDelta, type RegressionReport, REPORT_SCHEMA_VERSION, } from "./regression";
 import { AIEvalClient } from "./client";
 export default AIEvalClient;

package/dist/index.js

@@ -9,7 +9,7 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.extendExpectWithToPassGate = exports.Logger = exports.openAIChatEval = exports.traceOpenAI = exports.traceAnthropic = exports.runCheck = exports.parseArgs = exports.EXIT = exports.RequestCache = exports.CacheTTL = exports.RequestBatcher = exports.importData = exports.exportData = exports.compareSnapshots = exports.saveSnapshot = exports.compareWithSnapshot = exports.snapshot = exports.TestSuite = exports.createTestSuite = exports.ContextManager = exports.withContext = exports.getContext = exports.createContext = exports.withinRange = exports.similarTo = exports.respondedWithinTime = exports.notContainsPII = exports.matchesSchema = exports.matchesPattern = exports.isValidURL = exports.isValidEmail = exports.hasValidCodeSyntax = exports.hasSentiment = exports.hasReadabilityScore = exports.hasNoToxicity = exports.hasNoHallucinations = exports.hasLength = exports.hasFactualAccuracy = exports.followsInstructions = exports.expect = exports.containsLanguage = exports.containsKeywords = exports.containsJSON = exports.containsAllRequiredFields = exports.NetworkError = exports.ValidationError = exports.AuthenticationError = exports.RateLimitError = exports.EvalAIError = exports.AIEvalClient = void 0;
-exports.WorkflowTracer = exports.traceWorkflowStep = exports.traceLangChainAgent = exports.traceCrewAI = exports.traceAutoGen = exports.createWorkflowTracer = exports.EvaluationTemplates = exports.streamEvaluation = exports.RateLimiter = exports.batchRead = exports.batchProcess = exports.PaginatedIterator = exports.encodeCursor = exports.decodeCursor = exports.createPaginatedIterator = exports.autoPaginate = void 0;
+exports.REPORT_SCHEMA_VERSION = exports.GATE_EXIT = exports.GATE_CATEGORY = exports.ARTIFACTS = exports.WorkflowTracer = exports.traceWorkflowStep = exports.traceLangChainAgent = exports.traceCrewAI = exports.traceAutoGen = exports.createWorkflowTracer = exports.EvaluationTemplates = exports.streamEvaluation = exports.RateLimiter = exports.batchRead = exports.batchProcess = exports.PaginatedIterator = exports.encodeCursor = exports.decodeCursor = exports.createPaginatedIterator = exports.autoPaginate = void 0;
 // Main SDK exports
 var client_1 = require("./client");
 Object.defineProperty(exports, "AIEvalClient", { enumerable: true, get: function () { return client_1.AIEvalClient; } });
@@ -117,6 +117,12 @@ Object.defineProperty(exports, "traceCrewAI", { enumerable: true, get: function
 Object.defineProperty(exports, "traceLangChainAgent", { enumerable: true, get: function () { return workflows_1.traceLangChainAgent; } });
 Object.defineProperty(exports, "traceWorkflowStep", { enumerable: true, get: function () { return workflows_1.traceWorkflowStep; } });
 Object.defineProperty(exports, "WorkflowTracer", { enumerable: true, get: function () { return workflows_1.WorkflowTracer; } });
+// Regression gate constants & types (v1.6.0)
+var regression_1 = require("./regression");
+Object.defineProperty(exports, "ARTIFACTS", { enumerable: true, get: function () { return regression_1.ARTIFACTS; } });
+Object.defineProperty(exports, "GATE_CATEGORY", { enumerable: true, get: function () { return regression_1.GATE_CATEGORY; } });
+Object.defineProperty(exports, "GATE_EXIT", { enumerable: true, get: function () { return regression_1.GATE_EXIT; } });
+Object.defineProperty(exports, "REPORT_SCHEMA_VERSION", { enumerable: true, get: function () { return regression_1.REPORT_SCHEMA_VERSION; } });
 // Default export for convenience
 const client_2 = require("./client");
 exports.default = client_2.AIEvalClient;

package/dist/regression.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Regression gate constants and types.
+ *
+ * These mirror the contracts defined in scripts/regression-gate.ts
+ * and evals/schemas/regression-report.schema.json so that SDK consumers
+ * can programmatically inspect gate results without parsing strings.
+ *
+ * @packageDocumentation
+ */
+/** Exit codes emitted by `evalai gate` / `scripts/regression-gate.ts`. */
+export declare const GATE_EXIT: {
+    /** Gate passed — no regressions detected */
+    readonly PASS: 0;
+    /** One or more regression thresholds exceeded */
+    readonly REGRESSION: 1;
+    /** Infrastructure error (baseline missing, summary missing, etc.) */
+    readonly INFRA_ERROR: 2;
+    /** Confidence tests failed (test suite red) */
+    readonly CONFIDENCE_FAILED: 3;
+    /** Confidence summary file missing (test infra crashed) */
+    readonly CONFIDENCE_MISSING: 4;
+};
+export type GateExitCode = (typeof GATE_EXIT)[keyof typeof GATE_EXIT];
+/** Categories written to regression-report.json `category` field. */
+export declare const GATE_CATEGORY: {
+    readonly PASS: "pass";
+    readonly REGRESSION: "regression";
+    readonly INFRA_ERROR: "infra_error";
+};
+export type GateCategory = (typeof GATE_CATEGORY)[keyof typeof GATE_CATEGORY];
+/** Current schema version for regression-report.json. */
+export declare const REPORT_SCHEMA_VERSION = 1;
+export interface RegressionDelta {
+    metric: string;
+    baseline: number | string;
+    current: number | string;
+    delta: string;
+    status: "pass" | "fail";
+}
+export interface RegressionReport {
+    schemaVersion: number;
+    timestamp: string;
+    exitCode: GateExitCode;
+    category: GateCategory;
+    passed: boolean;
+    failures: string[];
+    deltas: RegressionDelta[];
+}
+export interface BaselineTolerance {
+    scoreDrop: number;
+    passRateDrop: number;
+    maxLatencyIncreaseMs: number;
+    maxCostIncreaseUsd: number;
+}
+export interface Baseline {
+    schemaVersion: number;
+    description: string;
+    generatedAt: string;
+    generatedBy: string;
+    commitSha: string;
+    updatedAt: string;
+    updatedBy: string;
+    tolerance: BaselineTolerance;
+    goldenEval: {
+        score: number;
+        passRate: number;
+        totalCases: number;
+        passedCases: number;
+    };
+    qualityScore: {
+        overall: number;
+        grade: string;
+        accuracy: number;
+        safety: number;
+        latency: number;
+        cost: number;
+        consistency: number;
+    };
+    confidenceTests: {
+        unitPassed: boolean;
+        unitTotal: number;
+        dbPassed: boolean;
+        dbTotal: number;
+    };
+    productMetrics: {
+        p95ApiLatencyMs?: number;
+        goldenCostUsd?: number;
+    };
+    qualityMetrics?: {
+        unitLaneDurationMs?: number;
+        dbLaneDurationMs?: number;
+    };
+}
+/** Well-known artifact paths relative to project root. */
+export declare const ARTIFACTS: {
+    readonly BASELINE: "evals/baseline.json";
+    readonly REGRESSION_REPORT: "evals/regression-report.json";
+    readonly CONFIDENCE_SUMMARY: "evals/confidence-summary.json";
+    readonly LATENCY_BENCHMARK: "evals/latency-benchmark.json";
+};

package/dist/regression.js

@@ -0,0 +1,44 @@
+"use strict";
+/**
+ * Regression gate constants and types.
+ *
+ * These mirror the contracts defined in scripts/regression-gate.ts
+ * and evals/schemas/regression-report.schema.json so that SDK consumers
+ * can programmatically inspect gate results without parsing strings.
+ *
+ * @packageDocumentation
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ARTIFACTS = exports.REPORT_SCHEMA_VERSION = exports.GATE_CATEGORY = exports.GATE_EXIT = void 0;
+// ── Exit codes ──
+/** Exit codes emitted by `evalai gate` / `scripts/regression-gate.ts`. */
+exports.GATE_EXIT = {
+    /** Gate passed — no regressions detected */
+    PASS: 0,
+    /** One or more regression thresholds exceeded */
+    REGRESSION: 1,
+    /** Infrastructure error (baseline missing, summary missing, etc.) */
+    INFRA_ERROR: 2,
+    /** Confidence tests failed (test suite red) */
+    CONFIDENCE_FAILED: 3,
+    /** Confidence summary file missing (test infra crashed) */
+    CONFIDENCE_MISSING: 4,
+};
+// ── Report categories ──
+/** Categories written to regression-report.json `category` field. */
+exports.GATE_CATEGORY = {
+    PASS: "pass",
+    REGRESSION: "regression",
+    INFRA_ERROR: "infra_error",
+};
+// ── Schema version ──
+/** Current schema version for regression-report.json. */
+exports.REPORT_SCHEMA_VERSION = 1;
+// ── Artifact paths ──
+/** Well-known artifact paths relative to project root. */
+exports.ARTIFACTS = {
+    BASELINE: "evals/baseline.json",
+    REGRESSION_REPORT: "evals/regression-report.json",
+    CONFIDENCE_SUMMARY: "evals/confidence-summary.json",
+    LATENCY_BENCHMARK: "evals/latency-benchmark.json",
+};

package/dist/version.d.ts

@@ -3,5 +3,5 @@
  * X-EvalAI-SDK-Version: SDK package version
  * X-EvalAI-Spec-Version: OpenAPI spec version (docs/openapi.json info.version)
  */
-export declare const SDK_VERSION = "1.5.0";
+export declare const SDK_VERSION = "1.6.0";
 export declare const SPEC_VERSION = "1.0.0";

package/dist/version.js

@@ -6,5 +6,5 @@ exports.SPEC_VERSION = exports.SDK_VERSION = void 0;
  * X-EvalAI-SDK-Version: SDK package version
  * X-EvalAI-Spec-Version: OpenAPI spec version (docs/openapi.json info.version)
  */
-exports.SDK_VERSION = "1.5.0";
+exports.SDK_VERSION = "1.6.0";
 exports.SPEC_VERSION = "1.0.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pauly4010/evalai-sdk",
-  "version": "1.5.8",
+  "version": "1.6.0",
   "description": "AI Evaluation Platform SDK - Complete API Coverage with Performance Optimizations",
   "main": "dist/index.js",
   "module": "dist/index.js",
@@ -102,6 +102,11 @@
       "import": "./dist/matchers/index.js",
       "require": "./dist/matchers/index.js",
       "types": "./dist/matchers/index.d.ts"
+    },
+    "./regression": {
+      "import": "./dist/regression.js",
+      "require": "./dist/regression.js",
+      "types": "./dist/regression.d.ts"
     }
   }
 }