@evalgate/sdk 2.0.0

package/dist/runtime/execution-mode.d.ts

@@ -0,0 +1,80 @@
+/**
+ * COMPAT-204: Dual-path execution toggle
+ *
+ * Environment flag EVALGATE_RUNTIME=legacy|spec|auto
+ * Auto uses spec runtime if manifest/specs exist, else legacy
+ * Existing projects continue unchanged; new projects can use DSL only
+ */
+/**
+ * Execution mode types
+ */
+export type ExecutionMode = "legacy" | "spec" | "auto";
+/**
+ * Execution mode configuration
+ */
+export interface ExecutionModeConfig {
+    /** Current execution mode */
+    mode: ExecutionMode;
+    /** Whether spec runtime is available */
+    hasSpecRuntime: boolean;
+    /** Whether legacy runtime is available */
+    hasLegacyRuntime: boolean;
+    /** Project root path */
+    projectRoot: string;
+    /** Detected spec files */
+    specFiles: string[];
+    /** Detected legacy config */
+    legacyConfig?: string;
+}
+/**
+ * Get execution mode from environment or auto-detection
+ */
+export declare function getExecutionMode(projectRoot?: string): Promise<ExecutionModeConfig>;
+/**
+ * Check if project can run in spec mode
+ */
+export declare function canRunSpecMode(config: ExecutionModeConfig): boolean;
+/**
+ * Check if project can run in legacy mode
+ */
+export declare function canRunLegacyMode(config: ExecutionModeConfig): boolean;
+/**
+ * Get recommended execution mode for project
+ */
+export declare function getRecommendedExecutionMode(config: ExecutionModeConfig): ExecutionMode;
+/**
+ * Validate execution mode compatibility
+ */
+export declare function validateExecutionMode(config: ExecutionModeConfig): {
+    valid: boolean;
+    warnings: string[];
+    errors: string[];
+};
+/**
+ * Print execution mode information
+ */
+export declare function printExecutionModeInfo(config: ExecutionModeConfig): void;
+/**
+ * Environment variable helpers
+ */
+export declare const ENV_VARS: {
+    readonly EXECUTION_MODE: "EVALGATE_RUNTIME";
+    readonly POSSIBLE_VALUES: readonly ["legacy", "spec", "auto"];
+    readonly DEFAULT: "auto";
+};
+/**
+ * Check if environment variable is set
+ */
+export declare function hasExecutionModeEnv(): boolean;
+/**
+ * Get current environment variable value
+ */
+export declare function getExecutionModeEnv(): string | undefined;
+/**
+ * Set execution mode environment variable
+ */
+export declare function setExecutionModeEnv(mode: ExecutionMode): void;
+/**
+ * Clear execution mode environment variable
+ */
+export declare function clearExecutionModeEnv(): void;

package/dist/runtime/execution-mode.js

@@ -0,0 +1,357 @@
+"use strict";
+/**
+ * COMPAT-204: Dual-path execution toggle
+ *
+ * Environment flag EVALGATE_RUNTIME=legacy|spec|auto
+ * Auto uses spec runtime if manifest/specs exist, else legacy
+ * Existing projects continue unchanged; new projects can use DSL only
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ENV_VARS = void 0;
+exports.getExecutionMode = getExecutionMode;
+exports.canRunSpecMode = canRunSpecMode;
+exports.canRunLegacyMode = canRunLegacyMode;
+exports.getRecommendedExecutionMode = getRecommendedExecutionMode;
+exports.validateExecutionMode = validateExecutionMode;
+exports.printExecutionModeInfo = printExecutionModeInfo;
+exports.hasExecutionModeEnv = hasExecutionModeEnv;
+exports.getExecutionModeEnv = getExecutionModeEnv;
+exports.setExecutionModeEnv = setExecutionModeEnv;
+exports.clearExecutionModeEnv = clearExecutionModeEnv;
+const fs = __importStar(require("node:fs/promises"));
+const path = __importStar(require("node:path"));
+/**
+ * Get execution mode from environment or auto-detection
+ */
+async function getExecutionMode(projectRoot = process.cwd()) {
+    // Check environment variable first
+    const envMode = process.env.EVALGATE_RUNTIME?.toLowerCase();
+    if (envMode === "legacy" || envMode === "spec" || envMode === "auto") {
+        return {
+            mode: envMode,
+            hasSpecRuntime: envMode !== "legacy",
+            hasLegacyRuntime: envMode !== "spec",
+            projectRoot,
+            specFiles: envMode !== "legacy" ? await findSpecFiles(projectRoot) : [],
+            legacyConfig: envMode !== "spec" ? await findLegacyConfig(projectRoot) : undefined,
+        };
+    }
+    // Auto-detect mode
+    return await autoDetectExecutionMode(projectRoot);
+}
+/**
+ * Auto-detect execution mode based on project structure
+ */
+async function autoDetectExecutionMode(projectRoot) {
+    const specFiles = await findSpecFiles(projectRoot);
+    const legacyConfig = await findLegacyConfig(projectRoot);
+    const hasSpecRuntime = specFiles.length > 0;
+    const hasLegacyRuntime = !!legacyConfig;
+    let mode = "auto";
+    // If both are available, prefer spec runtime for new projects
+    if (hasSpecRuntime && hasLegacyRuntime) {
+        mode = "spec"; // Prefer spec for mixed projects
+    }
+    else if (hasSpecRuntime) {
+        mode = "spec";
+    }
+    else if (hasLegacyRuntime) {
+        mode = "legacy";
+    }
+    else {
+        mode = "auto"; // Default to auto for empty projects
+    }
+    return {
+        mode,
+        hasSpecRuntime,
+        hasLegacyRuntime,
+        projectRoot,
+        specFiles,
+        legacyConfig,
+    };
+}
+/**
+ * Find spec files in project
+ */
+async function findSpecFiles(projectRoot) {
+    const specPatterns = [
+        "eval/**/*.spec.ts",
+        "eval/**/*.spec.js",
+        "src/**/*.spec.ts",
+        "src/**/*.spec.js",
+        "tests/**/*.spec.ts",
+        "tests/**/*.spec.js",
+        "spec/**/*.ts",
+        "spec/**/*.js",
+    ];
+    const foundFiles = [];
+    for (const pattern of specPatterns) {
+        try {
+            const files = await searchFiles(projectRoot, pattern, projectRoot);
+            foundFiles.push(...files);
+        }
+        catch (_error) {
+            // Ignore errors for non-existent paths
+        }
+    }
+    // Filter for files that contain defineEval calls
+    const specFilesWithDefineEval = [];
+    for (const file of foundFiles) {
+        try {
+            const content = await fs.readFile(file, "utf-8");
+            if (content.includes("defineEval")) {
+                specFilesWithDefineEval.push(file);
+            }
+        }
+        catch (_error) {
+            // Ignore read errors
+        }
+    }
+    return specFilesWithDefineEval;
+}
+/**
+ * Simple file search (placeholder for proper glob implementation)
+ */
+async function searchFiles(dir, pattern, projectRoot) {
+    const results = [];
+    try {
+        const entries = await fs.readdir(dir, { withFileTypes: true });
+        for (let i = 0; i < entries.length; i++) {
+            const entry = entries[i];
+            const fullPath = path.join(dir, entry.name);
+            if (entry.isDirectory() && !entry.name.startsWith(".")) {
+                results.push(...(await searchFiles(fullPath, pattern, projectRoot)));
+            }
+            else if (entry.isFile()) {
+                // Simple pattern matching
+                if (matchesPattern(fullPath, pattern, projectRoot)) {
+                    results.push(fullPath);
+                }
+            }
+        }
+    }
+    catch (_error) {
+        // Ignore permission errors
+    }
+    return results;
+}
+/**
+ * Simple pattern matching (placeholder for proper glob)
+ */
+function matchesPattern(filePath, pattern, projectRoot) {
+    const _fileName = path.basename(filePath);
+    const _ext = path.extname(filePath);
+    const _dir = path.dirname(filePath);
+    // Convert glob pattern to regex
+    // Handle **/ and * patterns correctly
+    let regexPattern = pattern;
+    // Replace **/ with (?:.*/)? to match optional directory path
+    regexPattern = regexPattern.replace(/\*\*\//g, "(?:.*/)?");
+    // Replace remaining * with [^/]* (filename pattern)
+    regexPattern = regexPattern.replace(/\*/g, "[^/]*");
+    const relativePath = path.relative(projectRoot, filePath).replace(/\\/g, "/");
+    const regex = new RegExp(`^${regexPattern}$`);
+    return regex.test(relativePath);
+}
+/**
+ * Find legacy config file
+ */
+async function findLegacyConfig(projectRoot) {
+    const configPaths = [
+        "evalai.config.json",
+        "evalai.config.js",
+        "evalai.config.ts",
+        ".evalgaterc",
+        ".evalgaterc.json",
+    ];
+    for (const configPath of configPaths) {
+        const fullPath = path.join(projectRoot, configPath);
+        try {
+            await fs.access(fullPath);
+            return fullPath;
+        }
+        catch (_error) {
+            // File doesn't exist, continue
+        }
+    }
+    return undefined;
+}
+/**
+ * Check if project can run in spec mode
+ */
+function canRunSpecMode(config) {
+    return config.hasSpecRuntime && config.specFiles.length > 0;
+}
+/**
+ * Check if project can run in legacy mode
+ */
+function canRunLegacyMode(config) {
+    return config.hasLegacyRuntime && !!config.legacyConfig;
+}
+/**
+ * Get recommended execution mode for project
+ */
+function getRecommendedExecutionMode(config) {
+    if (config.mode !== "auto") {
+        return config.mode;
+    }
+    if (canRunSpecMode(config) && canRunLegacyMode(config)) {
+        return "spec"; // Prefer spec for mixed projects
+    }
+    if (canRunSpecMode(config)) {
+        return "spec";
+    }
+    if (canRunLegacyMode(config)) {
+        return "legacy";
+    }
+    return "auto";
+}
+/**
+ * Validate execution mode compatibility
+ */
+function validateExecutionMode(config) {
+    const warnings = [];
+    const errors = [];
+    // Check for mixed project
+    if (config.hasSpecRuntime && config.hasLegacyRuntime) {
+        warnings.push("Project contains both spec files and legacy config. " +
+            "Consider migrating legacy tests to spec format for consistency.");
+    }
+    // Check for no runtime
+    if (!config.hasSpecRuntime && !config.hasLegacyRuntime) {
+        warnings.push("No tests found. This appears to be an empty project. " +
+            "Use 'evalgate init' to create a new project.");
+    }
+    // Check for spec mode without spec files
+    if (config.mode === "spec" && !canRunSpecMode(config)) {
+        errors.push("Spec mode requested but no spec files found. " +
+            "Create spec files with defineEval() or use legacy mode.");
+    }
+    // Check for legacy mode without config
+    if (config.mode === "legacy" && !canRunLegacyMode(config)) {
+        errors.push("Legacy mode requested but no evalgate.config.json (or evalai.config.json) found. " +
+            "Create a config file or use spec mode.");
+    }
+    return {
+        valid: errors.length === 0,
+        warnings,
+        errors,
+    };
+}
+/**
+ * Print execution mode information
+ */
+function printExecutionModeInfo(config) {
+    console.log(`🔧 EvalGate Execution Mode: ${config.mode.toUpperCase()}`);
+    console.log(`📁 Project root: ${config.projectRoot}`);
+    console.log(``);
+    if (config.hasSpecRuntime) {
+        console.log(`✅ Spec runtime available`);
+        console.log(`   Found ${config.specFiles.length} spec file(s):`);
+        config.specFiles.slice(0, 5).forEach((file) => {
+            console.log(`   - ${path.relative(config.projectRoot, file)}`);
+        });
+        if (config.specFiles.length > 5) {
+            console.log(`   ... and ${config.specFiles.length - 5} more`);
+        }
+    }
+    else {
+        console.log(`❌ No spec runtime found`);
+    }
+    console.log(``);
+    if (config.hasLegacyRuntime) {
+        console.log(`✅ Legacy runtime available`);
+        if (config.legacyConfig) {
+            console.log(`   Config: ${path.relative(config.projectRoot, config.legacyConfig)}`);
+        }
+    }
+    else {
+        console.log(`❌ No legacy runtime found`);
+    }
+    console.log(``);
+    const validation = validateExecutionMode(config);
+    if (validation.warnings.length > 0) {
+        console.log(`⚠️  Warnings:`);
+        validation.warnings.forEach((warning) => {
+            console.log(`   ${warning}`);
+        });
+        console.log(``);
+    }
+    if (validation.errors.length > 0) {
+        console.log(`❌ Errors:`);
+        validation.errors.forEach((error) => {
+            console.log(`   ${error}`);
+        });
+        console.log(``);
+    }
+    const recommended = getRecommendedExecutionMode(config);
+    console.log(`💡 Recommended mode: ${recommended.toUpperCase()}`);
+    if (config.mode === "auto") {
+        console.log(`🔄 Auto mode will use: ${recommended.toUpperCase()}`);
+    }
+}
+/**
+ * Environment variable helpers
+ */
+exports.ENV_VARS = {
+    EXECUTION_MODE: "EVALGATE_RUNTIME",
+    POSSIBLE_VALUES: ["legacy", "spec", "auto"],
+    DEFAULT: "auto",
+};
+/**
+ * Check if environment variable is set
+ */
+function hasExecutionModeEnv() {
+    return !!process.env.EVALGATE_RUNTIME;
+}
+/**
+ * Get current environment variable value
+ */
+function getExecutionModeEnv() {
+    return process.env.EVALGATE_RUNTIME;
+}
+/**
+ * Set execution mode environment variable
+ */
+function setExecutionModeEnv(mode) {
+    process.env.EVALGATE_RUNTIME = mode;
+}
+/**
+ * Clear execution mode environment variable
+ */
+function clearExecutionModeEnv() {
+    delete process.env.EVALGATE_RUNTIME;
+}

package/dist/runtime/executor.d.ts

@@ -0,0 +1,16 @@
+/**
+ * EvalGate Local Executor - Layer 1 Foundation
+ *
+ * Local execution engine that runs specifications without database coupling.
+ * Implements the execution interface for the new programming model.
+ */
+import type { LocalExecutor } from "./types";
+/**
+ * Create a new local executor
+ */
+export declare function createLocalExecutor(): LocalExecutor;
+/**
+ * Default local executor instance
+ * For convenience in simple use cases
+ */
+export declare const defaultLocalExecutor: LocalExecutor;

package/dist/runtime/executor.js

@@ -0,0 +1,152 @@
+"use strict";
+/**
+ * EvalGate Local Executor - Layer 1 Foundation
+ *
+ * Local execution engine that runs specifications without database coupling.
+ * Implements the execution interface for the new programming model.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultLocalExecutor = void 0;
+exports.createLocalExecutor = createLocalExecutor;
+const types_1 = require("./types");
+/**
+ * Local executor implementation
+ * Runs specifications in the current process without external dependencies
+ */
+class LocalExecutorImpl {
+    constructor() {
+        this.type = "local";
+        this.capabilities = {
+            type: "local",
+            parallel: true,
+            maxParallel: 4, // Conservative default
+            supportedModels: ["*"], // Supports any model via executor function
+            costTracking: false, // No built-in cost tracking for local execution
+            streaming: false, // No streaming for local execution
+        };
+    }
+    /**
+     * Execute a single specification with safe error boundaries
+     */
+    async executeSpec(spec, input) {
+        const startTime = Date.now();
+        let classification = "passed";
+        try {
+            // Create execution context
+            const context = {
+                input,
+                metadata: spec.metadata,
+                options: {
+                    model: spec.config?.model,
+                    budget: spec.config?.budget,
+                    timeout: spec.config?.timeout,
+                },
+            };
+            // Execute specification with timeout
+            const timeout = spec.config?.timeout || 30000; // 30 second default
+            const result = await this.executeWithTimeout(spec.executor, context, timeout);
+            // Add execution duration
+            result.durationMs = Date.now() - startTime;
+            // Determine classification
+            classification = result.pass ? "passed" : "failed";
+            return {
+                ...result,
+                classification,
+            };
+        }
+        catch (error) {
+            const duration = Date.now() - startTime;
+            let errorEnvelope;
+            if (error instanceof Error && error.name === "TimeoutError") {
+                classification = "timeout";
+                errorEnvelope = {
+                    classification: "timeout_error",
+                    code: "TIMEOUT",
+                    message: `Specification execution timed out after ${duration}ms`,
+                    stack: error.stack,
+                    testId: spec.id,
+                    filePath: spec.filePath,
+                    position: spec.position,
+                    timestamp: new Date().toISOString(),
+                };
+            }
+            else {
+                classification = "error";
+                // Create EvalExecutionError wrapper
+                const execError = new types_1.EvalExecutionError(error instanceof Error ? error.message : String(error), {
+                    testId: spec.id,
+                    filePath: spec.filePath,
+                    position: spec.position,
+                    code: "EXECUTION_ERROR",
+                    originalError: error instanceof Error ? error : undefined,
+                });
+                errorEnvelope = execError.toEnvelope();
+            }
+            return {
+                pass: false,
+                score: 0,
+                error: error instanceof Error ? error.message : String(error),
+                durationMs: duration,
+                classification,
+                errorEnvelope,
+            };
+        }
+    }
+    /**
+     * Execute multiple specifications
+     */
+    async executeSpecs(specs, inputs) {
+        if (specs.length !== inputs.length) {
+            throw new types_1.SpecExecutionError("Number of specs must match number of inputs", {
+                specCount: specs.length,
+                inputCount: inputs.length,
+            });
+        }
+        // For now, execute sequentially
+        // In Layer 3, we'll add parallel execution with worker pools
+        const results = [];
+        for (let i = 0; i < specs.length; i++) {
+            const result = await this.executeSpec(specs[i], inputs[i]);
+            results.push(result);
+        }
+        return results;
+    }
+    /**
+     * Get executor capabilities
+     */
+    getCapabilities() {
+        return this.capabilities;
+    }
+    /**
+     * Execute with timeout protection
+     */
+    async executeWithTimeout(fn, context, timeoutMs) {
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                const error = new Error(`Execution timed out after ${timeoutMs}ms`);
+                error.name = "TimeoutError";
+                reject(error);
+            }, timeoutMs);
+            fn(context)
+                .then((result) => {
+                clearTimeout(timer);
+                resolve(result);
+            })
+                .catch((error) => {
+                clearTimeout(timer);
+                reject(error);
+            });
+        });
+    }
+}
+/**
+ * Create a new local executor
+ */
+function createLocalExecutor() {
+    return new LocalExecutorImpl();
+}
+/**
+ * Default local executor instance
+ * For convenience in simple use cases
+ */
+exports.defaultLocalExecutor = createLocalExecutor();

package/dist/runtime/registry.d.ts

@@ -0,0 +1,78 @@
+/**
+ * EvalGate Runtime Registry - Layer 1 Foundation
+ *
+ * Scoped registry with proper lifecycle management.
+ * Prevents cross-run contamination and memory leaks.
+ */
+import type { EvalRuntime } from "./types";
+/**
+ * Runtime interface with lifecycle management
+ * Ensures proper cleanup and prevents resource leaks
+ */
+export interface RuntimeHandle {
+    /** Runtime instance */
+    runtime: EvalRuntime;
+    /** defineEval function bound to this runtime */
+    defineEval: typeof import("./eval").defineEval;
+    /** Dispose runtime and clean up resources */
+    dispose(): void;
+    /** Create runtime snapshot for persistence */
+    snapshot(): RuntimeSnapshot;
+    /** Load runtime from snapshot */
+    load(snapshot: RuntimeSnapshot): void;
+}
+/**
+ * Runtime snapshot for persistence and recovery
+ */
+export interface RuntimeSnapshot {
+    /** Runtime metadata */
+    runtimeId: string;
+    namespace: string;
+    createdAt: string;
+    /** Serialized specifications */
+    specs: SerializedSpec[];
+    /** Snapshot version for compatibility */
+    version: string;
+}
+/**
+ * Serialized specification for snapshot
+ */
+export interface SerializedSpec {
+    id: string;
+    name: string;
+    filePath: string;
+    position: {
+        line: number;
+        column: number;
+    };
+    description?: string;
+    tags?: string[];
+    metadata?: Record<string, unknown>;
+    config?: {
+        timeout?: number;
+        retries?: number;
+        budget?: string;
+        model?: string | "auto";
+    };
+    /** Serialized executor function (placeholder) */
+    executorSerialized: boolean;
+}
+/**
+ * Create a new scoped runtime with lifecycle management
+ * Returns a handle for proper resource management
+ */
+export declare function createEvalRuntime(projectRoot?: string): RuntimeHandle;
+/**
+ * Helper function for safe runtime execution with automatic cleanup
+ * Ensures runtime is disposed even if an exception is thrown
+ */
+export declare function withRuntime<T>(projectRoot: string, fn: (handle: RuntimeHandle) => Promise<T>): Promise<T>;
+export declare function getActiveRuntime(): EvalRuntime;
+/**
+ * Set the active runtime (for backward compatibility)
+ */
+export declare function setActiveRuntime(runtime: EvalRuntime): void;
+/**
+ * Dispose the active runtime (for backward compatibility)
+ */
+export declare function disposeActiveRuntime(): void;