@evalgate/sdk 2.0.0

package/dist/runtime/adapters/testsuite-to-dsl.js

@@ -0,0 +1,276 @@
+"use strict";
+/**
+ * COMPAT-202: Legacy TestSuite → defineEval adapter
+ *
+ * Converts legacy TestSuite instances to defineEval specifications
+ * without forcing migration. Enables lossless where possible.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.adaptTestSuite = adaptTestSuite;
+exports.generateDefineEvalCode = generateDefineEvalCode;
+const eval_1 = require("../eval");
+const registry_1 = require("../registry");
+/**
+ * Convert TestSuite to defineEval specifications
+ *
+ * @param suite - Legacy TestSuite instance
+ * @param options - Adapter configuration options
+ * @returns Array of EvalSpec definitions
+ */
+function adaptTestSuite(suite, options = {}) {
+    const { includeProvenance = true, preserveIds = true, generateHelpers = true, } = options;
+    // Get test suite data using the new getters
+    const tests = suite.getTests();
+    const metadata = suite.getMetadata();
+    const config = suite.getConfig();
+    // Create a temporary runtime for spec generation
+    const _runtime = (0, registry_1.createEvalRuntime)();
+    const specs = [];
+    try {
+        // Convert each test case to an EvalSpec
+        for (const test of tests) {
+            const spec = {
+                id: generateSpecId(test, metadata.suiteName || "legacy-suite", preserveIds),
+                name: test.id,
+                filePath: "legacy://testsuite", // Placeholder for legacy source
+                position: { line: 1, column: 1 }, // Placeholder position
+                description: `Legacy test: ${test.id}`,
+                tags: ["legacy", "migrated"],
+                executor: createExecutorFromTestCase(test, config, generateHelpers),
+                metadata: {
+                    ...test.metadata,
+                    ...(includeProvenance && {
+                        source: "legacy",
+                        legacySuiteName: metadata.suiteName,
+                        legacyTestId: test.id,
+                        originalInput: test.input,
+                        originalExpected: test.expected,
+                    }),
+                },
+                config: {
+                    timeout: config.timeout,
+                    retries: config.retries,
+                    // Note: budget, model not available in TestSuite
+                },
+            };
+            specs.push(spec);
+        }
+    }
+    finally {
+        // Clean up temporary runtime
+        (0, registry_1.disposeActiveRuntime)();
+    }
+    return specs;
+}
+/**
+ * Generate specification ID for legacy test
+ */
+function generateSpecId(test, suiteName, preserveIds) {
+    if (preserveIds && test.id && test.id !== `case-${test.id}`) {
+        // Use original ID if available and not auto-generated
+        return test.id.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 20);
+    }
+    // Generate deterministic ID from test content
+    const content = `${suiteName}|${test.id}|${test.input}|${test.expected || ""}`;
+    const hash = Buffer.from(content)
+        .toString("base64")
+        .replace(/[+/=]/g, "")
+        .slice(0, 20)
+        .toLowerCase();
+    return hash;
+}
+/**
+ * Create executor function from test case
+ */
+function createExecutorFromTestCase(test, config, generateHelpers) {
+    return async (context) => {
+        const input = context.input;
+        // If there's an executor in the config, use it
+        if (config.executor) {
+            try {
+                const output = await config.executor(input);
+                return evaluateTestCase(test, output, generateHelpers);
+            }
+            catch (error) {
+                return (0, eval_1.createResult)({
+                    pass: false,
+                    score: 0,
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+        // If there's an expected value, use it as output
+        if (test.expected !== undefined) {
+            return evaluateTestCase(test, test.expected, generateHelpers);
+        }
+        // No executor or expected value - this is an error case
+        return (0, eval_1.createResult)({
+            pass: false,
+            score: 0,
+            error: "No executor or expected output available for legacy test",
+        });
+    };
+}
+/**
+ * Evaluate test case against output
+ */
+function evaluateTestCase(test, output, generateHelpers) {
+    try {
+        let passed = true;
+        let score = 100;
+        const assertions = [];
+        // If there are assertions, run them
+        if (test.hasAssertions && test.assertionCount > 0) {
+            // Note: We can't actually run the assertions since they're functions
+            // In a real implementation, we'd need to serialize and execute them
+            // For now, we'll do basic validation
+            // Basic string comparison if expected is provided
+            if (test.expected !== undefined) {
+                const exactMatch = output === test.expected;
+                passed = exactMatch;
+                score = exactMatch ? 100 : 0;
+                assertions.push({
+                    name: "legacy-equals",
+                    passed: exactMatch,
+                    expected: test.expected,
+                    actual: output,
+                    message: exactMatch
+                        ? "Output matches expected"
+                        : `Expected "${test.expected}", got "${output}"`,
+                });
+            }
+        }
+        else {
+            // No assertions, assume pass if output exists
+            passed = output.length > 0;
+            score = passed ? 100 : 0;
+        }
+        return (0, eval_1.createResult)({
+            pass: passed,
+            score: score,
+            assertions: generateHelpers ? assertions : undefined,
+            metadata: {
+                testCaseId: test.id,
+                originalInput: test.input,
+                originalExpected: test.expected,
+            },
+        });
+    }
+    catch (error) {
+        return (0, eval_1.createResult)({
+            pass: false,
+            score: 0,
+            error: error instanceof Error ? error.message : String(error),
+        });
+    }
+}
+/**
+ * Generate defineEval code from TestSuite
+ *
+ * @param suite - Legacy TestSuite instance
+ * @param options - Code generation options
+ * @returns Generated TypeScript code
+ */
+function generateDefineEvalCode(suite, options = {}) {
+    const specs = adaptTestSuite(suite, options);
+    const metadata = suite.getMetadata();
+    const imports = [
+        `// Auto-generated from TestSuite: ${metadata.suiteName || "legacy-suite"}`,
+        `// Generated at: ${new Date().toISOString()}`,
+        `// This file replaces the legacy TestSuite with defineEval() specifications`,
+        "",
+        `import { defineEval, createResult } from '@evalgate/sdk';`,
+        "",
+    ];
+    const specCode = specs.map((spec, _index) => {
+        const helperCode = generateHelperFunctions(spec, options);
+        return [
+            `defineEval("${spec.name}", async (context) => {`,
+            `  // Legacy test input: ${JSON.stringify(spec.metadata?.originalInput)}`,
+            `  const input = context.input;`,
+            `  `,
+            `  // Legacy test execution`,
+            helperCode,
+            `  `,
+            `  // Legacy evaluation logic`,
+            `  const result = await evaluateLegacyTest(input, ${JSON.stringify(spec.metadata?.originalExpected)});`,
+            `  `,
+            `  return result;`,
+            `}, {`,
+            `  description: "${spec.description}",`,
+            `  tags: ${JSON.stringify(spec.tags)},`,
+            `  metadata: ${JSON.stringify(spec.metadata)},`,
+            `  timeout: ${spec.config?.timeout || 30000},`,
+            `  retries: ${spec.config?.retries || 0},`,
+            `});`,
+            "",
+        ].join("\n");
+    });
+    const helperFunctions = generateHelperFunctionsForSuite(specs, options);
+    const evaluationFunction = generateEvaluationFunction();
+    return [
+        ...imports,
+        ...helperFunctions,
+        ...evaluationFunction,
+        ...specCode,
+    ].join("\n");
+}
+/**
+ * Generate helper functions for a specific spec
+ */
+function generateHelperFunctions(spec, options) {
+    if (!options.generateHelpers)
+        return "";
+    // Generate helper functions based on test metadata
+    const helpers = [];
+    // Add helper for assertion evaluation if needed
+    if (spec.metadata?.originalExpected) {
+        helpers.push(`function evaluateLegacyAssertion(output: string, expected: string): boolean {`, `  return output === expected;`, `}`);
+    }
+    // Add helper for test evaluation
+    helpers.push(`async function evaluateLegacyTest(input: string, expected?: string): Promise<unknown> {`, `  // This function simulates the legacy test evaluation`, `  const output = await simulateLegacyExecutor(input);`, `  `, `  if (expected !== undefined) {`, `    const passed = evaluateLegacyAssertion(output, expected);`, `    return createResult({`, `      pass: passed,`, `      score: passed ? 100 : 0,`, `      metadata: {`, `        input,`, `        expected,`, `      },`, `    });`, `  }`, `  `, `  return createResult({`, `    pass: output.length > 0,`, `    score: output.length > 0 ? 100 : 0,`, `    metadata: { input },`, `  });`, `}`);
+    // Add executor simulation
+    helpers.push(`async function simulateLegacyExecutor(input: string): Promise<string> {`, `  // This function simulates the legacy executor`, `  // In a real migration, this would be replaced with the actual executor`, `  return input; // Echo for demonstration`, `}`);
+    return helpers.join("\n\n");
+}
+/**
+ * Generate helper functions for the entire suite
+ */
+function generateHelperFunctionsForSuite(specs, options) {
+    const helpers = new Set();
+    // Collect all unique helper functions needed
+    for (const spec of specs) {
+        const specHelpers = generateHelperFunctions(spec, options);
+        if (specHelpers) {
+            helpers.add(specHelpers);
+        }
+    }
+    return Array.from(helpers).join("\n\n");
+}
+/**
+ * Generate evaluation function
+ */
+function generateEvaluationFunction() {
+    return [
+        `// Legacy test evaluation function`,
+        `function evaluateLegacyTest(input: string, expected?: string): unknown {`,
+        `  // This function evaluates legacy test logic`,
+        `  // In a real migration, this would contain the actual test logic`,
+        `  `,
+        `  if (expected !== undefined) {`,
+        `    const passed = input === expected;`,
+        `    return createResult({`,
+        `      pass: passed,`,
+        `      score: passed ? 100 : 0,`,
+        `      metadata: { input, expected },`,
+        `    });`,
+        `  }`,
+        `  `,
+        `  return createResult({`,
+        `    pass: input.length > 0,`,
+        `    score: input.length > 0 ? 100 : 0,`,
+        `    metadata: { input },`,
+        `  });`,
+        `}`,
+    ].join("\n");
+}

package/dist/runtime/context.d.ts

@@ -0,0 +1,26 @@
+/**
+ * EvalGate Runtime Context - Layer 1 Foundation
+ *
+ * Execution context management for specifications.
+ * Provides clean isolation and proper resource management.
+ */
+import type { EvalContext, EvalOptions } from "./types";
+/**
+ * Create a new execution context
+ */
+export declare function createContext<TInput = string>(input: TInput, metadata?: Record<string, unknown>, options?: EvalOptions): EvalContext & {
+    input: TInput;
+};
+/**
+ * Merge contexts with proper precedence
+ * Later contexts override earlier ones
+ */
+export declare function mergeContexts(base: EvalContext, ...overrides: Partial<EvalContext>[]): EvalContext;
+/**
+ * Clone a context for safe modification
+ */
+export declare function cloneContext(context: EvalContext): EvalContext;
+/**
+ * Validate context structure
+ */
+export declare function validateContext(context: EvalContext): void;

package/dist/runtime/context.js

@@ -0,0 +1,74 @@
+"use strict";
+/**
+ * EvalGate Runtime Context - Layer 1 Foundation
+ *
+ * Execution context management for specifications.
+ * Provides clean isolation and proper resource management.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createContext = createContext;
+exports.mergeContexts = mergeContexts;
+exports.cloneContext = cloneContext;
+exports.validateContext = validateContext;
+/**
+ * Create a new execution context
+ */
+function createContext(input, metadata, options) {
+    return {
+        input: input,
+        metadata,
+        options,
+    };
+}
+/**
+ * Merge contexts with proper precedence
+ * Later contexts override earlier ones
+ */
+function mergeContexts(base, ...overrides) {
+    // Ensure base has a valid input
+    if (!base.input) {
+        throw new Error("Base context must have a valid input");
+    }
+    const merged = overrides.reduce((merged, override) => ({
+        input: override.input ?? merged.input,
+        metadata: {
+            ...merged.metadata,
+            ...override.metadata,
+        },
+        options: override.options
+            ? {
+                ...merged.options,
+                ...override.options,
+            }
+            : merged.options,
+    }), base);
+    // Type assertion since we've ensured input exists
+    return merged;
+}
+/**
+ * Clone a context for safe modification
+ */
+function cloneContext(context) {
+    return {
+        input: context.input,
+        metadata: { ...context.metadata },
+        options: context.options ? { ...context.options } : undefined,
+    };
+}
+/**
+ * Validate context structure
+ */
+function validateContext(context) {
+    if (!context || typeof context !== "object") {
+        throw new Error("Context must be an object");
+    }
+    if (typeof context.input !== "string") {
+        throw new Error("Context input must be a string");
+    }
+    if (context.metadata && typeof context.metadata !== "object") {
+        throw new Error("Context metadata must be an object");
+    }
+    if (context.options && typeof context.options !== "object") {
+        throw new Error("Context options must be an object");
+    }
+}

package/dist/runtime/eval.d.ts

@@ -0,0 +1,46 @@
+/**
+ * EvalGate defineEval() DSL - Layer 1 Foundation
+ *
+ * The core DSL function for defining behavioral specifications.
+ * Uses content-addressable identity with AST position for stability.
+ */
+import type { DefineEvalFunction, EvalContext, EvalResult } from "./types";
+/**
+ * Export the defineEval function with proper typing
+ * This is the main DSL entry point
+ */
+export declare const defineEval: DefineEvalFunction;
+/**
+ * Convenience export for evalai.test() alias (backward compatibility)
+ * Provides alternative naming that matches the original roadmap vision
+ */
+export declare const evalai: {
+    test: DefineEvalFunction;
+};
+/**
+ * Suite definition for grouping related specifications
+ * This will be expanded in Layer 3 for dependency graph support
+ */
+export declare function defineSuite(_name: string, specs: (() => void)[]): void;
+/**
+ * Helper function to create specification contexts
+ * Useful for testing and manual execution
+ */
+export declare function createContext<TInput = string>(input: TInput, metadata?: Record<string, unknown>, options?: EvalContext["options"]): EvalContext & {
+    input: TInput;
+};
+/**
+ * Helper function to create specification results
+ * Provides a convenient builder pattern for common result patterns
+ */
+export declare function createResult(config: {
+    pass: boolean;
+    score: number;
+    assertions?: EvalResult["assertions"];
+    metadata?: Record<string, unknown>;
+    error?: string;
+}): EvalResult;
+/**
+ * Default export for convenience
+ */
+export default defineEval;

package/dist/runtime/eval.js

@@ -0,0 +1,244 @@
+"use strict";
+/**
+ * EvalGate defineEval() DSL - Layer 1 Foundation
+ *
+ * The core DSL function for defining behavioral specifications.
+ * Uses content-addressable identity with AST position for stability.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evalai = exports.defineEval = void 0;
+exports.defineSuite = defineSuite;
+exports.createContext = createContext;
+exports.createResult = createResult;
+const crypto = __importStar(require("node:crypto"));
+const path = __importStar(require("node:path"));
+const registry_1 = require("./registry");
+const types_1 = require("./types");
+/**
+ * Extract AST position from call stack
+ * This provides stable identity that survives renames but changes when logic moves
+ */
+function getCallerPosition() {
+    const stack = new Error().stack;
+    if (!stack) {
+        throw new types_1.SpecRegistrationError("Unable to determine caller position");
+    }
+    // Parse stack trace to find the caller
+    const lines = stack.split("\n");
+    // Skip current function and find the actual caller
+    for (let i = 3; i < lines.length; i++) {
+        const line = lines[i];
+        if (!line ||
+            line.includes("node_modules") ||
+            line.includes("internal/modules")) {
+            continue;
+        }
+        // Extract file path, line, and column
+        const match = line.match(/at\s+.*?\((.*?):(\d+):(\d+)\)/);
+        if (match) {
+            const [, filePath, lineNum, colNum] = match;
+            return {
+                filePath: path.resolve(filePath),
+                line: parseInt(lineNum, 10),
+                column: parseInt(colNum, 10),
+            };
+        }
+        // Alternative format for some environments
+        const altMatch = line.match(/at\s+(.*?):(\d+):(\d+)/);
+        if (altMatch) {
+            const [, filePath, lineNum, colNum] = altMatch;
+            return {
+                filePath: path.resolve(filePath),
+                line: parseInt(lineNum, 10),
+                column: parseInt(colNum, 10),
+            };
+        }
+    }
+    throw new types_1.SpecRegistrationError("Unable to parse caller position from stack trace");
+}
+/**
+ * Generate content-addressable specification ID
+ */
+function generateSpecId(namespace, filePath, name, position) {
+    // Canonicalize path: relative to project root with POSIX separators
+    const projectRoot = process.cwd();
+    const relativePath = path.relative(projectRoot, filePath);
+    const canonicalPath = relativePath.split(path.sep).join("/"); // Force POSIX separators
+    const components = [
+        namespace,
+        canonicalPath,
+        name,
+        `${position.line}:${position.column}`,
+    ];
+    const content = components.join("|");
+    return crypto.createHash("sha256").update(content).digest("hex").slice(0, 20);
+}
+/**
+ * Validate specification name
+ */
+function validateSpecName(name) {
+    if (!name || typeof name !== "string") {
+        throw new types_1.SpecRegistrationError("Specification name must be a non-empty string");
+    }
+    if (name.trim() === "") {
+        throw new types_1.SpecRegistrationError("Specification name cannot be empty");
+    }
+    if (name.length > 100) {
+        throw new types_1.SpecRegistrationError("Specification name must be 100 characters or less");
+    }
+    // Check for invalid characters
+    if (!/^[a-zA-Z0-9\s\-_]+$/.test(name)) {
+        throw new types_1.SpecRegistrationError("Specification name can only contain letters, numbers, spaces, hyphens, and underscores");
+    }
+}
+/**
+ * Validate executor function
+ */
+function validateExecutor(executor) {
+    if (typeof executor !== "function") {
+        throw new types_1.SpecRegistrationError("Executor must be a function");
+    }
+    // Check function length (should accept context parameter)
+    if (executor.length > 1) {
+        throw new types_1.SpecRegistrationError("Executor should accept exactly one parameter (context)");
+    }
+}
+/**
+ * Create specification configuration from parameters
+ */
+function createSpecConfig(nameOrConfig, executor, options) {
+    if (typeof nameOrConfig === "string") {
+        // defineEval(name, executor, options) form
+        if (!executor) {
+            throw new types_1.SpecRegistrationError("Executor function is required when using name parameter");
+        }
+        return {
+            name: nameOrConfig,
+            executor,
+            ...options,
+        };
+    }
+    else {
+        // defineEval(config) form
+        return nameOrConfig;
+    }
+}
+/**
+ * Core defineEval function implementation
+ */
+function defineEvalImpl(nameOrConfig, executor, options) {
+    // Get caller position for identity
+    const callerPosition = getCallerPosition();
+    // Create specification configuration
+    const config = createSpecConfig(nameOrConfig, executor, options);
+    // Validate configuration
+    validateSpecName(config.name);
+    validateExecutor(config.executor);
+    // Get active runtime
+    const runtime = (0, registry_1.getActiveRuntime)();
+    // Generate specification ID
+    const specId = generateSpecId(runtime.namespace, callerPosition.filePath, config.name, callerPosition);
+    // Create specification
+    const spec = {
+        id: specId,
+        name: config.name,
+        filePath: callerPosition.filePath,
+        position: callerPosition,
+        description: config.description,
+        tags: config.tags,
+        executor: config.executor,
+        metadata: config.metadata,
+        config: {
+            timeout: config.timeout,
+            retries: config.retries,
+            budget: config.budget,
+            model: config.model,
+        },
+    };
+    // Register specification
+    runtime.register(spec);
+}
+/**
+ * Export the defineEval function with proper typing
+ * This is the main DSL entry point
+ */
+exports.defineEval = defineEvalImpl;
+/**
+ * Convenience export for evalai.test() alias (backward compatibility)
+ * Provides alternative naming that matches the original roadmap vision
+ */
+exports.evalai = {
+    test: exports.defineEval,
+};
+/**
+ * Suite definition for grouping related specifications
+ * This will be expanded in Layer 3 for dependency graph support
+ */
+function defineSuite(_name, specs) {
+    // For now, just execute the specs to register them
+    // In Layer 3, this will build the dependency graph
+    for (const specFn of specs) {
+        specFn();
+    }
+}
+/**
+ * Helper function to create specification contexts
+ * Useful for testing and manual execution
+ */
+function createContext(input, metadata, options) {
+    return {
+        input: input,
+        metadata,
+        options,
+    };
+}
+/**
+ * Helper function to create specification results
+ * Provides a convenient builder pattern for common result patterns
+ */
+function createResult(config) {
+    return {
+        pass: config.pass,
+        score: Math.max(0, Math.min(100, config.score)), // Clamp to 0-100
+        assertions: config.assertions,
+        metadata: config.metadata,
+        error: config.error,
+    };
+}
+/**
+ * Default export for convenience
+ */
+exports.default = exports.defineEval;