@evalgate/sdk 2.2.3 → 2.3.0

package/dist/runtime/eval.js

@@ -39,13 +39,21 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.evalai = exports.defineEval = void 0;
+exports.createLocalExecutor = exports.withRuntime = exports.setActiveRuntime = exports.getActiveRuntime = exports.disposeActiveRuntime = exports.createEvalRuntime = exports.evalai = exports.defineEval = void 0;
+exports.getFilteredSpecs = getFilteredSpecs;
 exports.defineSuite = defineSuite;
 exports.createContext = createContext;
+exports.createEvalContext = createContext;
 exports.createResult = createResult;
 const crypto = __importStar(require("node:crypto"));
+const fs = __importStar(require("node:fs"));
 const path = __importStar(require("node:path"));
 const registry_1 = require("./registry");
+Object.defineProperty(exports, "createEvalRuntime", { enumerable: true, get: function () { return registry_1.createEvalRuntime; } });
+Object.defineProperty(exports, "disposeActiveRuntime", { enumerable: true, get: function () { return registry_1.disposeActiveRuntime; } });
+Object.defineProperty(exports, "getActiveRuntime", { enumerable: true, get: function () { return registry_1.getActiveRuntime; } });
+Object.defineProperty(exports, "setActiveRuntime", { enumerable: true, get: function () { return registry_1.setActiveRuntime; } });
+Object.defineProperty(exports, "withRuntime", { enumerable: true, get: function () { return registry_1.withRuntime; } });
 const types_1 = require("./types");
 /**
  * Extract AST position from call stack
@@ -159,7 +167,7 @@ function createSpecConfig(nameOrConfig, executor, options) {
 /**
  * Core defineEval function implementation
  */
-function defineEvalImpl(nameOrConfig, executor, options) {
+function defineEvalWithMode(mode, nameOrConfig, executor, options) {
     // Get caller position for identity
     const callerPosition = getCallerPosition();
     // Create specification configuration
@@ -187,15 +195,124 @@ function defineEvalImpl(nameOrConfig, executor, options) {
             budget: config.budget,
             model: config.model,
         },
+        mode,
     };
     // Register specification
     runtime.register(spec);
 }
+function defineEvalImpl(nameOrConfig, executor, options) {
+    defineEvalWithMode("normal", nameOrConfig, executor, options);
+}
+function defineEvalSkipImpl(nameOrConfig, executor, options) {
+    defineEvalWithMode("skip", nameOrConfig, executor, options);
+}
+function defineEvalOnlyImpl(nameOrConfig, executor, options) {
+    defineEvalWithMode("only", nameOrConfig, executor, options);
+}
 /**
  * Export the defineEval function with proper typing
  * This is the main DSL entry point
  */
 exports.defineEval = defineEvalImpl;
+// Attach .skip and .only modifiers (vitest/jest convention)
+exports.defineEval.skip = defineEvalSkipImpl;
+exports.defineEval.only = defineEvalOnlyImpl;
+/**
+ * Parse a JSONL file into an array of row objects.
+ * Each line must be a valid JSON object; blank lines are skipped.
+ */
+function parseJsonl(content) {
+    return content
+        .split("\n")
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0)
+        .map((line, i) => {
+        try {
+            return JSON.parse(line);
+        }
+        catch {
+            throw new types_1.SpecRegistrationError(`Invalid JSON on line ${i + 1} of dataset`);
+        }
+    });
+}
+/**
+ * Parse a simple CSV file into an array of row objects.
+ * First line is treated as headers. Values are unquoted strings.
+ * For complex CSV (quoted fields, escapes), use a dedicated library.
+ */
+function parseCsv(content) {
+    const lines = content
+        .split("\n")
+        .map((l) => l.trim())
+        .filter((l) => l.length > 0);
+    if (lines.length < 2)
+        return [];
+    const headers = lines[0].split(",").map((h) => h.trim());
+    return lines.slice(1).map((line) => {
+        const values = line.split(",").map((v) => v.trim());
+        const row = {};
+        for (let i = 0; i < headers.length; i++) {
+            row[headers[i]] = values[i] ?? "";
+        }
+        return row;
+    });
+}
+/**
+ * Load a JSONL or CSV dataset and register one spec per row.
+ */
+function fromDatasetImpl(name, datasetPath, executor, options) {
+    const resolvedPath = path.isAbsolute(datasetPath)
+        ? datasetPath
+        : path.resolve(process.cwd(), datasetPath);
+    if (!fs.existsSync(resolvedPath)) {
+        throw new types_1.SpecRegistrationError(`Dataset file not found: ${resolvedPath}`);
+    }
+    const content = fs.readFileSync(resolvedPath, "utf8");
+    const ext = path.extname(resolvedPath).toLowerCase();
+    let rows;
+    if (ext === ".jsonl" || ext === ".ndjson") {
+        rows = parseJsonl(content);
+    }
+    else if (ext === ".csv") {
+        rows = parseCsv(content);
+    }
+    else if (ext === ".json") {
+        const parsed = JSON.parse(content);
+        rows = Array.isArray(parsed) ? parsed : [parsed];
+    }
+    else {
+        throw new types_1.SpecRegistrationError(`Unsupported dataset format: ${ext}. Use .jsonl, .ndjson, .csv, or .json`);
+    }
+    if (rows.length === 0) {
+        throw new types_1.SpecRegistrationError(`Dataset is empty: ${resolvedPath}`);
+    }
+    for (let i = 0; i < rows.length; i++) {
+        const row = rows[i];
+        const specName = `${name} - row ${i + 1}`;
+        const wrappedExecutor = (context) => executor({ ...context, input: row });
+        defineEvalWithMode("normal", specName, wrappedExecutor, {
+            ...options,
+            metadata: {
+                ...options?.metadata,
+                datasetPath: resolvedPath,
+                datasetRow: i + 1,
+            },
+        });
+    }
+}
+exports.defineEval.fromDataset = fromDatasetImpl;
+/**
+ * Filter a list of specs according to skip/only semantics:
+ * - If any spec has mode === "only", return only those specs
+ * - Otherwise, return all specs except those with mode === "skip"
+ */
+function getFilteredSpecs(specs) {
+    const onlySpecs = specs.filter((s) => s.mode === "only");
+    if (onlySpecs.length > 0) {
+        return onlySpecs;
+    }
+    return specs.filter((s) => s.mode !== "skip");
+}
 /**
  * Convenience export for evalai.test() alias (backward compatibility)
  * Provides alternative naming that matches the original roadmap vision
@@ -245,9 +362,17 @@ function createResult(config) {
         assertions: config.assertions,
         metadata: config.metadata,
         error: config.error,
+        output: config.output,
+        durationMs: config.durationMs,
+        tokens: config.tokens,
     };
 }
 /**
  * Default export for convenience
  */
+// Register defineEval with registry to break circular dependency
+(0, registry_1._registerDefineEval)(exports.defineEval);
+// Re-export createLocalExecutor from executor.ts
+var executor_1 = require("./executor");
+Object.defineProperty(exports, "createLocalExecutor", { enumerable: true, get: function () { return executor_1.createLocalExecutor; } });
 exports.default = exports.defineEval;

package/dist/runtime/registry.d.ts

@@ -4,7 +4,9 @@
  * Scoped registry with proper lifecycle management.
  * Prevents cross-run contamination and memory leaks.
  */
-import type { EvalRuntime } from "./types";
+import type { DefineEvalFunction, EvalRuntime } from "./types";
+/** @internal Called by eval.ts to register defineEval without circular import */
+export declare function _registerDefineEval(fn: (...args: unknown[]) => unknown): void;
 /**
  * Runtime interface with lifecycle management
  * Ensures proper cleanup and prevents resource leaks
@@ -13,7 +15,7 @@ export interface RuntimeHandle {
     /** Runtime instance */
     runtime: EvalRuntime;
     /** defineEval function bound to this runtime */
-    defineEval: typeof import("./eval").defineEval;
+    defineEval: DefineEvalFunction;
     /** Dispose runtime and clean up resources */
     dispose(): void;
     /** Create runtime snapshot for persistence */

package/dist/runtime/registry.js

@@ -39,6 +39,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
+exports._registerDefineEval = _registerDefineEval;
 exports.createEvalRuntime = createEvalRuntime;
 exports.withRuntime = withRuntime;
 exports.getActiveRuntime = getActiveRuntime;
@@ -47,6 +48,12 @@ exports.disposeActiveRuntime = disposeActiveRuntime;
 const crypto = __importStar(require("node:crypto"));
 const path = __importStar(require("node:path"));
 const types_1 = require("./types");
+// Registration pattern to break circular dependency (eval.ts imports from registry.ts)
+let _registeredDefineEval = null;
+/** @internal Called by eval.ts to register defineEval without circular import */
+function _registerDefineEval(fn) {
+    _registeredDefineEval = fn;
+}
 /**
  * Runtime registry implementation
  * Scoped lifecycle with proper memory management
@@ -326,9 +333,10 @@ function createEvalRuntime(projectRootOrConfig = process.cwd()) {
         const previousRuntime = activeRuntime;
         activeRuntime = runtime;
         try {
-            // Import and call defineEval
-            const { defineEval } = require("./eval");
-            return defineEval(nameOrConfig, executor, options);
+            if (!_registeredDefineEval) {
+                throw new types_1.RuntimeError("defineEval not registered. Ensure eval.ts is imported before calling createEvalRuntime.");
+            }
+            return _registeredDefineEval(nameOrConfig, executor, options);
         }
         finally {
             // Restore previous runtime

package/dist/runtime/run-report.d.ts

@@ -159,7 +159,7 @@ export declare class RunReportBuilder {
     addResult(testId: string, testName: string, filePath: string, position: {
         line: number;
         column: number;
-    }, input: string, result: EnhancedEvalResult): void;
+    }, input: string, result: EnhancedEvalResult, tags?: string[]): void;
     /**
      * Update summary statistics
      */

package/dist/runtime/run-report.js

@@ -77,7 +77,7 @@ class RunReportBuilder {
     /**
      * Add a test result to the report
      */
-    addResult(testId, testName, filePath, position, input, result) {
+    addResult(testId, testName, filePath, position, input, result, tags) {
         const runResult = {
             testId,
             testName,
@@ -88,7 +88,7 @@ class RunReportBuilder {
             score: result.score,
             durationMs: result.durationMs || 0,
             metadata: result.metadata,
-            tags: [], // TODO: Extract from spec
+            tags: tags ?? [],
             assertions: result.assertions?.map((assertion, index) => ({
                 name: assertion.name || `assertion-${index}`,
                 passed: assertion.passed,
@@ -182,8 +182,11 @@ class RunReportBuilder {
         // Set completion timestamp
         this.report.finishedAt = new Date().toISOString();
         const finalReport = this.report;
-        // Add toJSON method
-        finalReport.toJSON = () => JSON.stringify(finalReport, null, 2);
+        // Add toJSON method (spread to avoid circular reference via toJSON itself)
+        finalReport.toJSON = () => {
+            const { toJSON: _, ...data } = finalReport;
+            return JSON.stringify(data, null, 2);
+        };
         return finalReport;
     }
     /**

package/dist/runtime/types.d.ts

@@ -36,6 +36,8 @@ export interface EvalSpec {
         budget?: string;
         model?: string | "auto";
     };
+    /** Filtering mode: skip = registered but never executed, only = exclusive execution */
+    mode?: "normal" | "skip" | "only";
 }
 /**
  * Specification execution context
@@ -81,6 +83,10 @@ export interface EvalResult {
     durationMs?: number;
     /** Execution error if failed */
     error?: string;
+    /** Generated output text */
+    output?: string;
+    /** Token count consumed */
+    tokens?: number;
 }
 /**
  * Scoped runtime context - prevents cross-run contamination
@@ -183,6 +189,38 @@ export interface DefineEvalFunction {
      * @param config - Complete specification configuration
      */
     (config: SpecConfig): void;
+    /**
+     * Register a specification but skip it during execution.
+     * Follows the vitest/jest `.skip` convention.
+     */
+    skip: DefineEvalFunction;
+    /**
+     * Register a specification for exclusive execution.
+     * If any spec is marked `.only`, only those specs run.
+     * Follows the vitest/jest `.only` convention.
+     */
+    only: DefineEvalFunction;
+    /**
+     * Load a JSONL or CSV dataset and register one spec per row.
+     * Each row is passed as `context.input` (the parsed row object) to the executor.
+     *
+     * @param name - Base name for specs (each gets " [row N]" suffix)
+     * @param datasetPath - Path to a .jsonl or .csv file
+     * @param executor - Receives the parsed row as input
+     * @param options - Optional spec configuration applied to all rows
+     *
+     * @example
+     * ```ts
+     * defineEval.fromDataset("rag-accuracy", "./evals/golden.jsonl", async (ctx) => {
+     *   const row = ctx.input; // { question: string, expected: string }
+     *   const answer = await myRag(row.question);
+     *   return createResult({ pass: answer.includes(row.expected), score: 100 });
+     * });
+     * ```
+     */
+    fromDataset: <TRow extends Record<string, unknown> = Record<string, unknown>>(name: string, datasetPath: string, executor: (context: EvalContext & {
+        input: TRow;
+    }) => Promise<EvalResult>, options?: SpecOptions) => void;
 }
 /**
  * Specification definition options

package/dist/testing.d.ts

@@ -51,8 +51,16 @@ export interface TestSuiteConfig {
     stopOnFailure?: boolean;
     /** Timeout per test case in ms (default: 30000) */
     timeout?: number;
+    /** Alias for stopOnFailure — fail the entire suite on the first failing case. Useful in pre-commit hooks. */
+    strict?: boolean;
     /** Retry failing cases N times (default: 0). Only failing cases are retried. */
     retries?: number;
+    /** Base delay between retries in ms (default: 500). Exponential backoff: delay * 2^attempt. */
+    retryDelayMs?: number;
+    /** Add random jitter up to this fraction of the delay (default: 0.5 = ±50%). Set 0 to disable. */
+    retryJitter?: number;
+    /** Seed for deterministic case ordering. When set, cases are shuffled using this seed for reproducible runs. */
+    seed?: number;
 }
 export interface TestSuiteCaseResult {
     /** Test case ID */

package/dist/testing.js

@@ -50,6 +50,26 @@ class TestSuite {
     async run() {
         const startTime = Date.now();
         const results = [];
+        // Deterministic shuffle when seed is provided
+        const orderedCases = this.config.cases.map((c, i) => ({
+            case: c,
+            originalIndex: i,
+        }));
+        if (this.config.seed !== undefined) {
+            // mulberry32 seeded PRNG
+            let s = this.config.seed | 0;
+            const rand = () => {
+                s = (s + 0x6d2b79f5) | 0;
+                let t = Math.imul(s ^ (s >>> 15), 1 | s);
+                t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+                return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+            };
+            // Fisher-Yates shuffle
+            for (let i = orderedCases.length - 1; i > 0; i--) {
+                const j = Math.floor(rand() * (i + 1));
+                [orderedCases[i], orderedCases[j]] = [orderedCases[j], orderedCases[i]];
+            }
+        }
         const runTestCase = async (testCase, index) => {
             const caseStartTime = Date.now();
             const id = testCase.id || `case-${index}`;
@@ -114,37 +134,52 @@ class TestSuite {
                 };
             }
         };
-        // Run tests
+        // Run tests (using orderedCases which may be seeded-shuffled)
         if (this.config.parallel) {
-            results.push(...(await Promise.all(this.config.cases.map((tc, i) => runTestCase(tc, i)))));
+            results.push(...(await Promise.all(orderedCases.map((oc) => runTestCase(oc.case, oc.originalIndex)))));
         }
         else {
-            for (let i = 0; i < this.config.cases.length; i++) {
-                const result = await runTestCase(this.config.cases[i], i);
+            for (const oc of orderedCases) {
+                const result = await runTestCase(oc.case, oc.originalIndex);
                 results.push(result);
-                if (this.config.stopOnFailure && !result.passed) {
+                if ((this.config.stopOnFailure || this.config.strict) &&
+                    !result.passed) {
                     break;
                 }
             }
         }
         const retriedCases = [];
         const retries = this.config.retries ?? 0;
+        const baseDelay = this.config.retryDelayMs ?? 500;
+        const jitterFraction = this.config.retryJitter ?? 0.5;
         if (retries > 0 && results.length > 0) {
             const failingIndices = results
                 .map((r, i) => (r.passed ? -1 : i))
                 .filter((i) => i >= 0);
             for (let attempt = 0; attempt < retries && failingIndices.length > 0; attempt++) {
+                // Exponential backoff with jitter before each retry round
+                const delay = baseDelay * 2 ** attempt;
+                const jitter = jitterFraction > 0
+                    ? delay * jitterFraction * (Math.random() * 2 - 1)
+                    : 0;
+                const waitMs = Math.max(0, Math.round(delay + jitter));
+                if (waitMs > 0) {
+                    await new Promise((resolve) => setTimeout(resolve, waitMs));
+                }
                 const toRetry = [...failingIndices];
                 failingIndices.length = 0;
-                for (const i of toRetry) {
-                    const tc = this.config.cases[i];
-                    const retryResult = await runTestCase(tc, i);
+                for (const idx of toRetry) {
+                    const tc = results[idx]; // retry based on result index
+                    const originalCase = orderedCases.find((oc) => (oc.case.id || `case-${oc.originalIndex}`) === tc.id);
+                    if (!originalCase)
+                        continue;
+                    const retryResult = await runTestCase(originalCase.case, originalCase.originalIndex);
                     if (retryResult.passed) {
-                        results[i] = retryResult;
+                        results[idx] = retryResult;
                         retriedCases.push(retryResult.id);
                     }
                     else {
-                        failingIndices.push(i);
+                        failingIndices.push(idx);
                     }
                 }
             }

package/dist/version.d.ts

@@ -3,5 +3,5 @@
  * X-EvalGate-SDK-Version: SDK package version
  * X-EvalGate-Spec-Version: OpenAPI spec version (docs/openapi.json info.version)
  */
-export declare const SDK_VERSION = "2.2.3";
-export declare const SPEC_VERSION = "2.2.3";
+export declare const SDK_VERSION = "2.3.0";
+export declare const SPEC_VERSION = "2.3.0";

package/dist/version.js

@@ -6,5 +6,5 @@ exports.SPEC_VERSION = exports.SDK_VERSION = void 0;
  * X-EvalGate-SDK-Version: SDK package version
  * X-EvalGate-Spec-Version: OpenAPI spec version (docs/openapi.json info.version)
  */
-exports.SDK_VERSION = "2.2.3";
-exports.SPEC_VERSION = "2.2.3";
+exports.SDK_VERSION = "2.3.0";
+exports.SPEC_VERSION = "2.3.0";

package/dist/workflows.d.ts

@@ -170,6 +170,8 @@ export interface WorkflowTracerOptions {
     captureFullPayloads?: boolean;
     /** Debug mode */
     debug?: boolean;
+    /** Offline mode — skip all API calls, keep in-memory state only */
+    offline?: boolean;
 }
 /**
  * Agent span context