@evalgate/sdk 2.2.2 → 2.2.4

package/dist/cli/validate.js

@@ -0,0 +1,252 @@
+"use strict";
+/**
+ * evalgate validate — static validation of spec files without execution
+ *
+ * The equivalent of `tsc --noEmit` for eval specs. Catches:
+ * - Missing or malformed defineEval calls
+ * - Executor functions that don't return EvalResult shape
+ * - Invalid spec names (characters, length)
+ * - Empty spec files
+ * - Missing required fields in config-form defineEval
+ *
+ * Usage:
+ *   evalgate validate
+ *   evalgate validate --format json
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runValidate = runValidate;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const execution_mode_1 = require("../runtime/execution-mode");
+/**
+ * Name validation regex — must match the runtime's validateSpecName
+ */
+const VALID_NAME_RE = /^[a-zA-Z0-9\s\-_]+$/;
+const MAX_NAME_LENGTH = 100;
+/**
+ * Static patterns we look for in spec files
+ */
+const DEFINE_EVAL_RE = /defineEval\s*[.(]/g;
+const DEFINE_EVAL_NAME_RE = /defineEval\s*\(\s*["'`]([^"'`]*)["'`]/g;
+const DEFINE_EVAL_CONFIG_RE = /defineEval\s*\(\s*\{/g;
+const DEFINE_EVAL_SKIP_RE = /defineEval\.skip\s*\(/g;
+const DEFINE_EVAL_ONLY_RE = /defineEval\.only\s*\(/g;
+const DEFINE_EVAL_FROM_DATASET_RE = /defineEval\.fromDataset\s*\(/g;
+const EXECUTOR_RETURN_RE = /return\s*\{[^}]*pass\s*:/g;
+const CREATE_RESULT_RE = /createResult\s*\(/g;
+function analyzeFile(filePath) {
+    const issues = [];
+    const relPath = path.relative(process.cwd(), filePath);
+    let content;
+    try {
+        content = fs.readFileSync(filePath, "utf8");
+    }
+    catch {
+        issues.push({
+            severity: "error",
+            file: relPath,
+            code: "FILE_UNREADABLE",
+            message: `Cannot read file: ${relPath}`,
+        });
+        return issues;
+    }
+    if (content.trim().length === 0) {
+        issues.push({
+            severity: "error",
+            file: relPath,
+            code: "EMPTY_FILE",
+            message: "Spec file is empty",
+        });
+        return issues;
+    }
+    const lines = content.split("\n");
+    // Check for defineEval calls
+    const defineEvalMatches = content.match(DEFINE_EVAL_RE);
+    const skipMatches = content.match(DEFINE_EVAL_SKIP_RE);
+    const onlyMatches = content.match(DEFINE_EVAL_ONLY_RE);
+    const fromDatasetMatches = content.match(DEFINE_EVAL_FROM_DATASET_RE);
+    const totalCalls = (defineEvalMatches?.length ?? 0) +
+        (skipMatches?.length ?? 0) +
+        (onlyMatches?.length ?? 0) +
+        (fromDatasetMatches?.length ?? 0);
+    if (totalCalls === 0) {
+        issues.push({
+            severity: "warn",
+            file: relPath,
+            code: "NO_DEFINE_EVAL",
+            message: "No defineEval() calls found. File may not define any specs.",
+        });
+    }
+    // Validate spec names
+    const nameMatches = [...content.matchAll(DEFINE_EVAL_NAME_RE)];
+    for (const match of nameMatches) {
+        const name = match[1];
+        const matchIndex = match.index ?? 0;
+        const lineNum = content.substring(0, matchIndex).split("\n").length;
+        if (!name || name.trim() === "") {
+            issues.push({
+                severity: "error",
+                file: relPath,
+                line: lineNum,
+                code: "EMPTY_NAME",
+                message: "Spec name is empty",
+            });
+            continue;
+        }
+        if (name.length > MAX_NAME_LENGTH) {
+            issues.push({
+                severity: "error",
+                file: relPath,
+                line: lineNum,
+                code: "NAME_TOO_LONG",
+                message: `Spec name "${name.slice(0, 30)}..." exceeds ${MAX_NAME_LENGTH} characters`,
+            });
+        }
+        if (!VALID_NAME_RE.test(name)) {
+            issues.push({
+                severity: "error",
+                file: relPath,
+                line: lineNum,
+                code: "INVALID_NAME",
+                message: `Spec name "${name}" contains invalid characters (only letters, numbers, spaces, hyphens, underscores allowed)`,
+            });
+        }
+    }
+    // Check config-form defineEval calls have required fields
+    const configMatches = [...content.matchAll(DEFINE_EVAL_CONFIG_RE)];
+    for (const match of configMatches) {
+        const matchIndex = match.index ?? 0;
+        const lineNum = content.substring(0, matchIndex).split("\n").length;
+        // Simple heuristic: look for 'name:' and 'executor:' in the next ~20 lines
+        const contextLines = lines.slice(lineNum - 1, lineNum + 19).join("\n");
+        if (!contextLines.includes("name:") && !contextLines.includes("name :")) {
+            issues.push({
+                severity: "error",
+                file: relPath,
+                line: lineNum,
+                code: "MISSING_NAME",
+                message: "Config-form defineEval() missing required 'name' field",
+            });
+        }
+        if (!contextLines.includes("executor:") &&
+            !contextLines.includes("executor :")) {
+            issues.push({
+                severity: "error",
+                file: relPath,
+                line: lineNum,
+                code: "MISSING_EXECUTOR",
+                message: "Config-form defineEval() missing required 'executor' field",
+            });
+        }
+    }
+    // Check that executors return EvalResult shape
+    const hasCreateResult = CREATE_RESULT_RE.test(content);
+    const hasReturnPass = EXECUTOR_RETURN_RE.test(content);
+    if (totalCalls > 0 && !hasCreateResult && !hasReturnPass) {
+        issues.push({
+            severity: "warn",
+            file: relPath,
+            code: "NO_RESULT_SHAPE",
+            message: "No createResult() or return { pass: ... } found. Executors may not return the required EvalResult shape.",
+        });
+    }
+    return issues;
+}
+async function runValidate(args = []) {
+    const formatIndex = args.indexOf("--format");
+    const format = formatIndex !== -1 ? args[formatIndex + 1] : "human";
+    const projectRoot = process.cwd();
+    const executionMode = await (0, execution_mode_1.getExecutionMode)(projectRoot);
+    const specFiles = executionMode.specFiles;
+    if (specFiles.length === 0) {
+        const result = {
+            filesScanned: 0,
+            filesWithIssues: 0,
+            issues: [],
+            passed: true,
+        };
+        if (format === "json") {
+            console.log(JSON.stringify(result, null, 2));
+        }
+        else {
+            console.log("\n✨ No spec files found. Nothing to validate.");
+            console.log("💡 Create files with defineEval() calls to get started.");
+        }
+        return result;
+    }
+    const allIssues = [];
+    const filesWithIssues = new Set();
+    for (const file of specFiles) {
+        const issues = analyzeFile(file);
+        for (const issue of issues) {
+            allIssues.push(issue);
+            filesWithIssues.add(issue.file);
+        }
+    }
+    const errors = allIssues.filter((i) => i.severity === "error");
+    const warnings = allIssues.filter((i) => i.severity === "warn");
+    const passed = errors.length === 0;
+    const result = {
+        filesScanned: specFiles.length,
+        filesWithIssues: filesWithIssues.size,
+        issues: allIssues,
+        passed,
+    };
+    if (format === "json") {
+        console.log(JSON.stringify(result, null, 2));
+    }
+    else {
+        console.log(`\n🔍 Validated ${specFiles.length} spec file${specFiles.length === 1 ? "" : "s"}`);
+        if (allIssues.length === 0) {
+            console.log("✅ All spec files are valid.\n");
+        }
+        else {
+            for (const issue of allIssues) {
+                const loc = issue.line ? `:${issue.line}` : "";
+                const icon = issue.severity === "error" ? "❌" : "⚠️";
+                console.log(`  ${icon} ${issue.file}${loc} [${issue.code}] ${issue.message}`);
+            }
+            console.log(`\n${errors.length} error${errors.length === 1 ? "" : "s"}, ${warnings.length} warning${warnings.length === 1 ? "" : "s"}`);
+            if (passed) {
+                console.log("✅ Validation passed (warnings only).\n");
+            }
+            else {
+                console.log("❌ Validation failed.\n");
+            }
+        }
+    }
+    return result;
+}

package/dist/cli/watch.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Watch mode for evalgate run
+ *
+ * Re-executes evaluation specs when source files change.
+ * Uses Node.js fs.watch with debouncing to avoid rapid re-runs.
+ */
+import type { RunOptions } from "./run";
+export interface WatchOptions extends RunOptions {
+    /** Debounce interval in milliseconds (default: 300) */
+    debounceMs?: number;
+    /** Additional directories to watch beyond spec files */
+    extraWatchDirs?: string[];
+    /** Clear terminal between runs */
+    clearScreen?: boolean;
+}
+/**
+ * Start watch mode — runs evaluations and re-runs on file changes
+ */
+export declare function runWatch(options: WatchOptions, projectRoot?: string): Promise<void>;

package/dist/cli/watch.js

@@ -0,0 +1,175 @@
+"use strict";
+/**
+ * Watch mode for evalgate run
+ *
+ * Re-executes evaluation specs when source files change.
+ * Uses Node.js fs.watch with debouncing to avoid rapid re-runs.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runWatch = runWatch;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const run_1 = require("./run");
+/**
+ * Start watch mode — runs evaluations and re-runs on file changes
+ */
+async function runWatch(options, projectRoot = process.cwd()) {
+    const debounceMs = options.debounceMs ?? 300;
+    const clearScreen = options.clearScreen ?? true;
+    // Directories to watch
+    const watchDirs = new Set();
+    // Always watch the eval/ directory if it exists
+    const evalDir = path.join(projectRoot, "eval");
+    if (fs.existsSync(evalDir))
+        watchDirs.add(evalDir);
+    // Watch evals/ directory too
+    const evalsDir = path.join(projectRoot, "evals");
+    if (fs.existsSync(evalsDir))
+        watchDirs.add(evalsDir);
+    // Watch src/ for code changes that may affect evals
+    const srcDir = path.join(projectRoot, "src");
+    if (fs.existsSync(srcDir))
+        watchDirs.add(srcDir);
+    // Add extra watch dirs
+    if (options.extraWatchDirs) {
+        for (const dir of options.extraWatchDirs) {
+            const resolved = path.isAbsolute(dir) ? dir : path.join(projectRoot, dir);
+            if (fs.existsSync(resolved))
+                watchDirs.add(resolved);
+        }
+    }
+    if (watchDirs.size === 0) {
+        console.error("❌ No directories to watch. Create eval/, evals/, or src/ directory.");
+        process.exit(1);
+    }
+    console.log("👁️  Watch mode enabled");
+    console.log(`   Watching: ${[...watchDirs].map((d) => path.relative(projectRoot, d) || ".").join(", ")}`);
+    console.log(`   Debounce: ${debounceMs}ms`);
+    console.log("   Press Ctrl+C to stop\n");
+    // Initial run
+    await executeRun(options, projectRoot, clearScreen, false);
+    // Set up watchers with debouncing
+    let debounceTimer = null;
+    let isRunning = false;
+    const triggerRun = () => {
+        if (debounceTimer)
+            clearTimeout(debounceTimer);
+        debounceTimer = setTimeout(async () => {
+            if (isRunning)
+                return;
+            isRunning = true;
+            try {
+                await executeRun(options, projectRoot, clearScreen, true);
+            }
+            finally {
+                isRunning = false;
+            }
+        }, debounceMs);
+    };
+    const watchers = [];
+    for (const dir of watchDirs) {
+        try {
+            const watcher = fs.watch(dir, { recursive: true }, (eventType, filename) => {
+                if (!filename)
+                    return;
+                // Skip hidden files and node_modules
+                if (filename.startsWith(".") || filename.includes("node_modules"))
+                    return;
+                // Only watch relevant file types
+                const ext = path.extname(filename).toLowerCase();
+                if ([".ts", ".tsx", ".js", ".jsx", ".json", ".jsonl", ".csv"].includes(ext)) {
+                    console.log(`\n🔄 Change detected: ${filename} (${eventType})`);
+                    triggerRun();
+                }
+            });
+            watchers.push(watcher);
+        }
+        catch (err) {
+            console.warn(`⚠️  Could not watch ${path.relative(projectRoot, dir)}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    // Handle graceful shutdown
+    const cleanup = () => {
+        console.log("\n\n👋 Watch mode stopped.");
+        for (const watcher of watchers) {
+            watcher.close();
+        }
+        if (debounceTimer)
+            clearTimeout(debounceTimer);
+        process.exit(0);
+    };
+    process.on("SIGINT", cleanup);
+    process.on("SIGTERM", cleanup);
+    // Keep process alive
+    await new Promise(() => {
+        // Never resolves — watch runs until interrupted
+    });
+}
+/**
+ * Execute a single run and print results (without process.exit)
+ */
+async function executeRun(options, projectRoot, clearScreen, isRerun) {
+    if (clearScreen && isRerun) {
+        // Clear screen using ANSI escape
+        process.stdout.write("\x1B[2J\x1B[0f");
+    }
+    const timestamp = new Date().toLocaleTimeString();
+    console.log(`${isRerun ? "🔄 Re-running" : "▶️  Running"} evaluations... (${timestamp})`);
+    try {
+        const result = await (0, run_1.runEvaluations)({
+            specIds: options.specIds,
+            impactedOnly: options.impactedOnly,
+            baseBranch: options.baseBranch,
+            format: options.format,
+            writeResults: options.writeResults,
+        }, projectRoot);
+        if (options.format === "json") {
+            (0, run_1.printJsonResults)(result);
+        }
+        else {
+            (0, run_1.printHumanResults)(result);
+        }
+        // Print watch-specific summary
+        const statusIcon = result.summary.failed > 0 ? "❌" : "✅";
+        console.log(`\n${statusIcon} ${result.summary.passed}/${result.results.length} passed | Waiting for changes...`);
+        return result;
+    }
+    catch (error) {
+        console.error("❌ Run failed:", error instanceof Error ? error.message : String(error));
+        console.log("\n⏳ Waiting for changes...");
+        return null;
+    }
+}

package/dist/client.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.AIEvalClient = void 0;
 const batch_1 = require("./batch");
 const cache_1 = require("./cache");
+const constants_1 = require("./constants");
 const context_1 = require("./context");
 const errors_1 = require("./errors");
 const logger_1 = require("./logger");
@@ -72,7 +73,7 @@ class AIEvalClient {
         this.baseUrl =
             config.baseUrl ||
                 getEnvVar("EVALGATE_BASE_URL", "EVALAI_BASE_URL") ||
-                (isBrowser ? "" : "http://localhost:3000");
+                (isBrowser ? "" : constants_1.DEFAULT_BASE_URL);
         this.timeout = config.timeout || 30000;
         // Tier 4.17: Debug mode with request logging
         const logLevel = config.logLevel || (config.debug ? "debug" : "info");
@@ -100,7 +101,7 @@ class AIEvalClient {
             const MAX_CONCURRENCY = 5;
             this.batcher = new batch_1.RequestBatcher(async (requests) => {
                 const results = [];
-                const executing = [];
+                const executing = new Set();
                 for (const req of requests) {
                     const task = (async () => {
                         try {
@@ -121,18 +122,10 @@ class AIEvalClient {
                             });
                         }
                     })();
-                    executing.push(task);
-                    if (executing.length >= MAX_CONCURRENCY) {
+                    const tracked = task.finally(() => executing.delete(tracked));
+                    executing.add(tracked);
+                    if (executing.size >= MAX_CONCURRENCY) {
                         await Promise.race(executing);
-                        // Remove settled promises
-                        for (let i = executing.length - 1; i >= 0; i--) {
-                            const settled = await Promise.race([
-                                executing[i].then(() => true),
-                                Promise.resolve(false),
-                            ]);
-                            if (settled)
-                                executing.splice(i, 1);
-                        }
                     }
                 }
                 await Promise.allSettled(executing);

package/dist/constants.d.ts

@@ -0,0 +1,2 @@
+/** Default API base URL for the EvalGate platform. */
+export declare const DEFAULT_BASE_URL = "https://api.evalgate.com";

package/dist/constants.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_BASE_URL = void 0;
+/** Default API base URL for the EvalGate platform. */
+exports.DEFAULT_BASE_URL = "https://api.evalgate.com";

package/dist/errors.js

@@ -271,6 +271,10 @@ class RateLimitError extends EvalGateError {
     constructor(message, retryAfter) {
         super(message, "RATE_LIMIT_EXCEEDED", 429, { retryAfter });
         this.name = "RateLimitError";
+        if (retryAfter !== undefined) {
+            this.retryAfter = retryAfter;
+        }
+        Object.setPrototypeOf(this, RateLimitError.prototype);
     }
 }
 exports.RateLimitError = RateLimitError;
@@ -278,6 +282,7 @@ class AuthenticationError extends EvalGateError {
     constructor(message = "Authentication failed") {
         super(message, "AUTHENTICATION_ERROR", 401);
         this.name = "AuthenticationError";
+        Object.setPrototypeOf(this, AuthenticationError.prototype);
     }
 }
 exports.AuthenticationError = AuthenticationError;
@@ -285,6 +290,7 @@ class ValidationError extends EvalGateError {
     constructor(message = "Validation failed", details) {
         super(message, "VALIDATION_ERROR", 400, details);
         this.name = "ValidationError";
+        Object.setPrototypeOf(this, ValidationError.prototype);
     }
 }
 exports.ValidationError = ValidationError;
@@ -293,6 +299,7 @@ class NetworkError extends EvalGateError {
         super(message, "NETWORK_ERROR", 0);
         this.name = "NetworkError";
         this.retryable = true;
+        Object.setPrototypeOf(this, NetworkError.prototype);
     }
 }
 exports.NetworkError = NetworkError;

package/dist/export.js

@@ -155,7 +155,7 @@ async function importData(client, data, options = {}) {
         return result;
     }
     // Import traces
-    if (data.traces) {
+    if (data.traces && client?.traces) {
         const traceResults = { imported: 0, skipped: 0, failed: 0 };
         for (const trace of data.traces) {
             try {
@@ -191,7 +191,7 @@ async function importData(client, data, options = {}) {
         result.summary.total += data.traces.length;
     }
     // Import evaluations
-    if (data.evaluations) {
+    if (data.evaluations && client?.evaluations) {
         const evalResults = { imported: 0, skipped: 0, failed: 0 };
         for (const evaluation of data.evaluations) {
             try {

package/dist/index.d.ts

@@ -7,34 +7,35 @@
  * @packageDocumentation
  */
 export { AIEvalClient } from "./client";
-import { AuthenticationError, EvalGateError, NetworkError, RateLimitError, SDKError } from "./errors";
-export { EvalGateError, RateLimitError, AuthenticationError, SDKError as ValidationError, // Using SDKError as ValidationError for backward compatibility
-NetworkError, };
-export { type AssertionLLMConfig, configureAssertions, containsAllRequiredFields, containsJSON, containsKeywords, containsLanguage, containsLanguageAsync, expect, followsInstructions, getAssertionConfig, hasFactualAccuracy, hasFactualAccuracyAsync, hasLength, hasNoHallucinations, hasNoHallucinationsAsync, hasNoToxicity, hasNoToxicityAsync, hasPII, hasReadabilityScore, hasSentiment, hasSentimentAsync, hasValidCodeSyntax, hasValidCodeSyntaxAsync, isValidEmail, isValidURL, matchesPattern, matchesSchema, notContainsPII, respondedWithinTime, similarTo, withinRange, } from "./assertions";
+import { AuthenticationError, EvalGateError, NetworkError, RateLimitError, ValidationError } from "./errors";
+export { EvalGateError, RateLimitError, AuthenticationError, ValidationError, NetworkError, };
+export { type AssertionLLMConfig, configureAssertions, containsAllRequiredFields, containsJSON, containsKeywords, containsLanguage, containsLanguageAsync, expect, followsInstructions, getAssertionConfig, hasConsistency, hasConsistencyAsync, hasFactualAccuracy, hasFactualAccuracyAsync, hasLength, hasNoHallucinations, hasNoHallucinationsAsync, hasNoToxicity, hasNoToxicityAsync, hasPII, hasReadabilityScore, hasSentiment, hasSentimentAsync, hasSentimentWithScore, hasValidCodeSyntax, hasValidCodeSyntaxAsync, isValidEmail, isValidURL, matchesPattern, matchesSchema, notContainsPII, respondedWithinDuration, respondedWithinTime, respondedWithinTimeSince, type SentimentAsyncResult, similarTo, toSemanticallyContain, withinRange, } from "./assertions";
+export { EvalGateError as SDKError } from "./errors";
 import { createContext, EvalContext, getCurrentContext, withContext } from "./context";
 export { createContext, getCurrentContext as getContext, withContext, EvalContext as ContextManager, };
 export { cloneContext, mergeContexts, validateContext, } from "./runtime/context";
-export { createContext as createEvalContext, createResult, defineEval, defineSuite, evalai, } from "./runtime/eval";
+export { createContext as createEvalContext, createResult, defineEval, defineSuite, evalai, getFilteredSpecs, } from "./runtime/eval";
 export { createLocalExecutor, defaultLocalExecutor, } from "./runtime/executor";
 export { createEvalRuntime, disposeActiveRuntime, getActiveRuntime, setActiveRuntime, } from "./runtime/registry";
 export type { CloudExecutor, DefineEvalFunction, EvalContext, EvalExecutor, EvalExecutorInterface, EvalOptions, EvalResult, EvalRuntime, EvalSpec, ExecutorCapabilities, LocalExecutor, SpecConfig, SpecOptions, WorkerExecutor, } from "./runtime/types";
 export { EvalRuntimeError, RuntimeError, SpecExecutionError, SpecRegistrationError, } from "./runtime/types";
 export { createTestSuite, type TestCaseResult, TestSuite, TestSuiteCase, TestSuiteCaseResult, TestSuiteConfig, TestSuiteResult, } from "./testing";
-import { compareWithSnapshot, snapshot } from "./snapshot";
-export { snapshot, compareWithSnapshot, snapshot as saveSnapshot, compareWithSnapshot as compareSnapshots, };
+import { compareSnapshots, compareWithSnapshot, snapshot } from "./snapshot";
+export { snapshot, compareWithSnapshot, compareSnapshots, snapshot as saveSnapshot, };
 import type { ExportFormat } from "./export";
 import { exportData, importData } from "./export";
 export { exportData, importData };
 export type { ExportFormat, ExportFormat as ExportType };
 export { RequestBatcher } from "./batch";
-export { CacheTTL, RequestCache } from "./cache";
+export { CacheTTL } from "./cache";
 export { type CheckArgs, EXIT, parseArgs, runCheck } from "./cli/check";
 export { traceAnthropic } from "./integrations/anthropic";
 export { traceOpenAI } from "./integrations/openai";
 export { type OpenAIChatEvalCase, type OpenAIChatEvalOptions, type OpenAIChatEvalResult, openAIChatEval, } from "./integrations/openai-eval";
 export { Logger } from "./logger";
 export { extendExpectWithToPassGate } from "./matchers";
-export { autoPaginate, createPaginatedIterator, decodeCursor, encodeCursor, PaginatedIterator, type PaginatedResponse, type PaginationParams, } from "./pagination";
+export { createOTelExporter, type OTelAttribute, type OTelEvent, OTelExporter, type OTelExporterOptions, type OTelExportPayload, type OTelSpan, } from "./otel";
+export { autoPaginate, autoPaginateGenerator, createPaginatedIterator, decodeCursor, encodeCursor, PaginatedIterator, type PaginatedResponse, type PaginationParams, } from "./pagination";
 export { ARTIFACTS, type Baseline, type BaselineTolerance, GATE_CATEGORY, GATE_EXIT, type GateCategory, type GateExitCode, REPORT_SCHEMA_VERSION, type RegressionDelta, type RegressionReport, } from "./regression";
 export { batchProcess, batchRead, RateLimiter, streamEvaluation, } from "./streaming";
 export type { Annotation, AnnotationItem, AnnotationTask, APIKey, APIKeyUsage, APIKeyWithSecret, BatchOptions, ClientConfig as AIEvalConfig, CreateAnnotationItemParams, CreateAnnotationParams, CreateAnnotationTaskParams, CreateAPIKeyParams, CreateLLMJudgeConfigParams, CreateWebhookParams, Evaluation as EvaluationData, EvaluationRun, EvaluationRunDetail, ExportOptions, GenericMetadata as AnnotationData, GetLLMJudgeAlignmentParams, GetUsageParams, ImportOptions, ListAnnotationItemsParams, ListAnnotationsParams, ListAnnotationTasksParams, ListAPIKeysParams, ListLLMJudgeConfigsParams, ListLLMJudgeResultsParams, ListWebhookDeliveriesParams, ListWebhooksParams, LLMJudgeAlignment, LLMJudgeConfig, LLMJudgeEvaluateResult, LLMJudgeResult as LLMJudgeData, Organization, RetryConfig, SnapshotData, Span as SpanData, StreamOptions, TestCase, TestResult, Trace as TraceData, TraceDetail, TracedResponse, UpdateAPIKeyParams, UpdateWebhookParams, UsageStats, UsageSummary, Webhook, WebhookDelivery, } from "./types";