@evalgate/sdk 2.0.0

package/dist/snapshot.d.ts

@@ -0,0 +1,176 @@
+/**
+ * Snapshot Testing System
+ * Tier 4.16: Visual regression detection for LLM outputs
+ *
+ * ⚠️ NOTE: This module requires Node.js and will not work in browsers.
+ *
+ * @example
+ * ```typescript
+ * import { snapshot, loadSnapshot } from '@ai-eval-platform/sdk';
+ *
+ * const output = await generateText('Write a haiku about coding');
+ * await snapshot(output, 'haiku-test');
+ *
+ * // Later, compare with snapshot
+ * const saved = await loadSnapshot('haiku-test');
+ * const matches = compareSnapshots(saved, output);
+ * ```
+ */
+export interface SnapshotMetadata {
+    /** Snapshot name/ID */
+    name: string;
+    /** When snapshot was created */
+    createdAt: string;
+    /** Content hash for change detection */
+    hash: string;
+    /** Optional tags for organization */
+    tags?: string[];
+    /** Additional metadata */
+    metadata?: Record<string, unknown>;
+}
+export interface SnapshotData {
+    /** The actual output that was snapshotted */
+    output: string;
+    /** Metadata about the snapshot */
+    metadata: SnapshotMetadata;
+}
+export interface SnapshotComparison {
+    /** Whether snapshots match */
+    matches: boolean;
+    /** Similarity score 0-1 */
+    similarity: number;
+    /** Differences found */
+    differences: string[];
+    /** Original snapshot */
+    original: string;
+    /** New output */
+    current: string;
+}
+/**
+ * Snapshot manager
+ */
+export declare class SnapshotManager {
+    private snapshotDir;
+    constructor(snapshotDir?: string);
+    /**
+     * Ensure snapshot directory exists
+     */
+    private ensureSnapshotDir;
+    /**
+     * Get snapshot file path with security checks
+     */
+    private getSnapshotPath;
+    /**
+     * Generate content hash
+     */
+    private generateHash;
+    /**
+     * Save a snapshot
+     *
+     * @example
+     * ```typescript
+     * const manager = new SnapshotManager();
+     * await manager.save('haiku-test', output, { tags: ['poetry'] });
+     * ```
+     */
+    save(name: string, output: string, options?: {
+        tags?: string[];
+        metadata?: Record<string, unknown>;
+        overwrite?: boolean;
+    }): Promise<SnapshotData>;
+    /**
+     * Load a snapshot
+     *
+     * @example
+     * ```typescript
+     * const snapshot = await manager.load('haiku-test');
+     * console.log(snapshot.output);
+     * ```
+     */
+    load(name: string): Promise<SnapshotData>;
+    /**
+     * Compare current output with saved snapshot
+     *
+     * @example
+     * ```typescript
+     * const comparison = await manager.compare('haiku-test', currentOutput);
+     * if (!comparison.matches) {
+     *   console.log('Differences:', comparison.differences);
+     * }
+     * ```
+     */
+    compare(name: string, currentOutput: string): Promise<SnapshotComparison>;
+    /**
+     * List all snapshots
+     *
+     * @example
+     * ```typescript
+     * const snapshots = await manager.list();
+     * snapshots.forEach(s => console.log(s.metadata.name));
+     * ```
+     */
+    list(): Promise<SnapshotData[]>;
+    /**
+     * Delete a snapshot
+     *
+     * @example
+     * ```typescript
+     * await manager.delete('old-test');
+     * ```
+     */
+    delete(name: string): Promise<void>;
+    /**
+     * Update a snapshot with new output
+     *
+     * @example
+     * ```typescript
+     * await manager.update('haiku-test', newOutput);
+     * ```
+     */
+    update(name: string, output: string): Promise<SnapshotData>;
+}
+/**
+ * Save a snapshot (convenience function)
+ *
+ * @example
+ * ```typescript
+ * const output = await generateText('Write a haiku');
+ * await snapshot(output, 'haiku-test');
+ * ```
+ */
+export declare function snapshot(output: string, name: string, options?: {
+    tags?: string[];
+    metadata?: Record<string, unknown>;
+    overwrite?: boolean;
+    dir?: string;
+}): Promise<SnapshotData>;
+/**
+ * Load a snapshot (convenience function)
+ *
+ * @example
+ * ```typescript
+ * const saved = await loadSnapshot('haiku-test');
+ * console.log(saved.output);
+ * ```
+ */
+export declare function loadSnapshot(name: string, dir?: string): Promise<SnapshotData>;
+/**
+ * Compare with snapshot (convenience function)
+ *
+ * @example
+ * ```typescript
+ * const comparison = await compareWithSnapshot('haiku-test', currentOutput);
+ * if (!comparison.matches) {
+ *   console.log('Output changed!');
+ * }
+ * ```
+ */
+export declare function compareWithSnapshot(name: string, currentOutput: string, dir?: string): Promise<SnapshotComparison>;
+/**
+ * Delete a snapshot (convenience function)
+ */
+export declare function deleteSnapshot(name: string, dir?: string): Promise<void>;
+/**
+ * List all snapshots (convenience function)
+ */
+export declare function listSnapshots(dir?: string): Promise<SnapshotData[]>;

package/dist/snapshot.js

@@ -0,0 +1,322 @@
+"use strict";
+/**
+ * Snapshot Testing System
+ * Tier 4.16: Visual regression detection for LLM outputs
+ *
+ * ⚠️ NOTE: This module requires Node.js and will not work in browsers.
+ *
+ * @example
+ * ```typescript
+ * import { snapshot, loadSnapshot } from '@ai-eval-platform/sdk';
+ *
+ * const output = await generateText('Write a haiku about coding');
+ * await snapshot(output, 'haiku-test');
+ *
+ * // Later, compare with snapshot
+ * const saved = await loadSnapshot('haiku-test');
+ * const matches = compareSnapshots(saved, output);
+ * ```
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SnapshotManager = void 0;
+exports.snapshot = snapshot;
+exports.loadSnapshot = loadSnapshot;
+exports.compareWithSnapshot = compareWithSnapshot;
+exports.deleteSnapshot = deleteSnapshot;
+exports.listSnapshots = listSnapshots;
+// Environment check
+const isNode = typeof process !== "undefined" && process.versions?.node;
+if (!isNode) {
+    throw new Error("Snapshot testing requires Node.js and cannot run in browsers. " +
+        "This feature uses the filesystem for storing snapshots.");
+}
+const crypto = __importStar(require("node:crypto"));
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+/**
+ * Snapshot manager
+ */
+class SnapshotManager {
+    constructor(snapshotDir = "./.snapshots") {
+        this.snapshotDir = snapshotDir;
+        this.ensureSnapshotDir();
+    }
+    /**
+     * Ensure snapshot directory exists
+     */
+    ensureSnapshotDir() {
+        if (!fs.existsSync(this.snapshotDir)) {
+            fs.mkdirSync(this.snapshotDir, { recursive: true });
+        }
+    }
+    /**
+     * Get snapshot file path with security checks
+     */
+    getSnapshotPath(name) {
+        // Security: prevent empty names
+        if (!name || name.trim().length === 0) {
+            throw new Error("Snapshot name cannot be empty");
+        }
+        // Security: prevent path traversal
+        if (name.includes("..") || name.includes("/") || name.includes("\\")) {
+            throw new Error('Snapshot name cannot contain path separators or ".."');
+        }
+        // Sanitize to alphanumeric, hyphens, and underscores
+        const sanitized = name.replace(/[^a-zA-Z0-9-_]/g, "-");
+        // Security: ensure sanitized name is not empty
+        if (sanitized.length === 0) {
+            throw new Error("Snapshot name must contain at least one alphanumeric character");
+        }
+        // Security: prevent absolute paths
+        const filePath = path.join(this.snapshotDir, `${sanitized}.json`);
+        const resolvedPath = path.resolve(filePath);
+        const resolvedDir = path.resolve(this.snapshotDir);
+        if (!resolvedPath.startsWith(resolvedDir)) {
+            throw new Error("Invalid snapshot path: path traversal detected");
+        }
+        return filePath;
+    }
+    /**
+     * Generate content hash
+     */
+    generateHash(content) {
+        return crypto.createHash("sha256").update(content).digest("hex");
+    }
+    /**
+     * Save a snapshot
+     *
+     * @example
+     * ```typescript
+     * const manager = new SnapshotManager();
+     * await manager.save('haiku-test', output, { tags: ['poetry'] });
+     * ```
+     */
+    async save(name, output, options) {
+        const filePath = this.getSnapshotPath(name);
+        // Check if snapshot exists
+        if (!options?.overwrite && fs.existsSync(filePath)) {
+            throw new Error(`Snapshot '${name}' already exists. Use overwrite: true to update.`);
+        }
+        const snapshotData = {
+            output,
+            metadata: {
+                name,
+                createdAt: new Date().toISOString(),
+                hash: this.generateHash(output),
+                tags: options?.tags,
+                metadata: options?.metadata,
+            },
+        };
+        fs.writeFileSync(filePath, JSON.stringify(snapshotData, null, 2));
+        return snapshotData;
+    }
+    /**
+     * Load a snapshot
+     *
+     * @example
+     * ```typescript
+     * const snapshot = await manager.load('haiku-test');
+     * console.log(snapshot.output);
+     * ```
+     */
+    async load(name) {
+        const filePath = this.getSnapshotPath(name);
+        if (!fs.existsSync(filePath)) {
+            throw new Error(`Snapshot '${name}' not found`);
+        }
+        const content = fs.readFileSync(filePath, "utf-8");
+        return JSON.parse(content);
+    }
+    /**
+     * Compare current output with saved snapshot
+     *
+     * @example
+     * ```typescript
+     * const comparison = await manager.compare('haiku-test', currentOutput);
+     * if (!comparison.matches) {
+     *   console.log('Differences:', comparison.differences);
+     * }
+     * ```
+     */
+    async compare(name, currentOutput) {
+        const snapshot = await this.load(name);
+        const original = snapshot.output;
+        // Exact match check
+        const exactMatch = original === currentOutput;
+        // Calculate similarity (simple line-based diff)
+        const originalLines = original.split("\n");
+        const currentLines = currentOutput.split("\n");
+        const differences = [];
+        const maxLines = Math.max(originalLines.length, currentLines.length);
+        let matchingLines = 0;
+        for (let i = 0; i < maxLines; i++) {
+            const origLine = originalLines[i] || "";
+            const currLine = currentLines[i] || "";
+            if (origLine === currLine) {
+                matchingLines++;
+            }
+            else {
+                differences.push(`Line ${i + 1}: "${origLine}" → "${currLine}"`);
+            }
+        }
+        const similarity = maxLines > 0 ? matchingLines / maxLines : 1;
+        return {
+            matches: exactMatch,
+            similarity,
+            differences,
+            original,
+            current: currentOutput,
+        };
+    }
+    /**
+     * List all snapshots
+     *
+     * @example
+     * ```typescript
+     * const snapshots = await manager.list();
+     * snapshots.forEach(s => console.log(s.metadata.name));
+     * ```
+     */
+    async list() {
+        const files = fs.readdirSync(this.snapshotDir);
+        const snapshots = [];
+        for (const file of files) {
+            if (file.endsWith(".json")) {
+                const content = fs.readFileSync(path.join(this.snapshotDir, file), "utf-8");
+                snapshots.push(JSON.parse(content));
+            }
+        }
+        return snapshots;
+    }
+    /**
+     * Delete a snapshot
+     *
+     * @example
+     * ```typescript
+     * await manager.delete('old-test');
+     * ```
+     */
+    async delete(name) {
+        const filePath = this.getSnapshotPath(name);
+        if (!fs.existsSync(filePath)) {
+            throw new Error(`Snapshot '${name}' not found`);
+        }
+        fs.unlinkSync(filePath);
+    }
+    /**
+     * Update a snapshot with new output
+     *
+     * @example
+     * ```typescript
+     * await manager.update('haiku-test', newOutput);
+     * ```
+     */
+    async update(name, output) {
+        const existing = await this.load(name);
+        return this.save(name, output, {
+            tags: existing.metadata.tags,
+            metadata: existing.metadata.metadata,
+            overwrite: true,
+        });
+    }
+}
+exports.SnapshotManager = SnapshotManager;
+// Global snapshot manager instance
+let globalManager;
+/**
+ * Get or create global snapshot manager
+ */
+function getSnapshotManager(dir) {
+    if (!globalManager || dir) {
+        globalManager = new SnapshotManager(dir);
+    }
+    return globalManager;
+}
+/**
+ * Save a snapshot (convenience function)
+ *
+ * @example
+ * ```typescript
+ * const output = await generateText('Write a haiku');
+ * await snapshot(output, 'haiku-test');
+ * ```
+ */
+async function snapshot(output, name, options) {
+    const manager = getSnapshotManager(options?.dir);
+    return manager.save(name, output, options);
+}
+/**
+ * Load a snapshot (convenience function)
+ *
+ * @example
+ * ```typescript
+ * const saved = await loadSnapshot('haiku-test');
+ * console.log(saved.output);
+ * ```
+ */
+async function loadSnapshot(name, dir) {
+    const manager = getSnapshotManager(dir);
+    return manager.load(name);
+}
+/**
+ * Compare with snapshot (convenience function)
+ *
+ * @example
+ * ```typescript
+ * const comparison = await compareWithSnapshot('haiku-test', currentOutput);
+ * if (!comparison.matches) {
+ *   console.log('Output changed!');
+ * }
+ * ```
+ */
+async function compareWithSnapshot(name, currentOutput, dir) {
+    const manager = getSnapshotManager(dir);
+    return manager.compare(name, currentOutput);
+}
+/**
+ * Delete a snapshot (convenience function)
+ */
+async function deleteSnapshot(name, dir) {
+    const manager = getSnapshotManager(dir);
+    return manager.delete(name);
+}
+/**
+ * List all snapshots (convenience function)
+ */
+async function listSnapshots(dir) {
+    const manager = getSnapshotManager(dir);
+    return manager.list();
+}

package/dist/streaming.d.ts

@@ -0,0 +1,173 @@
+/**
+ * Streaming & Batch Operations
+ * Tier 2.8: Handle large datasets efficiently
+ *
+ * @example
+ * ```typescript
+ * import { streamEvaluations, batchCreate } from '@ai-eval-platform/sdk';
+ *
+ * // Stream large evaluation results
+ * for await (const result of streamEvaluations(client, config)) {
+ *   console.log(`Progress: ${result.completed}/${result.total}`);
+ * }
+ *
+ * // Batch create traces
+ * await batchCreate(client.traces, traces, { batchSize: 100 });
+ * ```
+ */
+export interface BatchOptions {
+    /** Batch size (default: 100) */
+    batchSize?: number;
+    /** Run batches in parallel (default: true) */
+    parallel?: boolean;
+    /** Delay between batches in ms (default: 0) */
+    delayMs?: number;
+    /** Callback for progress updates */
+    onProgress?: (progress: BatchProgress) => void;
+    /** Callback for batch errors */
+    onError?: (error: BatchError) => void;
+    /** Continue on error (default: true) */
+    continueOnError?: boolean;
+}
+export interface BatchProgress {
+    /** Total items */
+    total: number;
+    /** Completed items */
+    completed: number;
+    /** Failed items */
+    failed: number;
+    /** Current batch number */
+    batch: number;
+    /** Total batches */
+    totalBatches: number;
+}
+export interface BatchError {
+    /** Batch number where error occurred */
+    batch: number;
+    /** Item index in batch */
+    index: number;
+    /** The error */
+    error: Error;
+    /** The item that failed */
+    item: unknown;
+}
+export interface BatchResult<T> {
+    /** Successfully processed items */
+    successful: T[];
+    /** Failed items */
+    failed: Array<{
+        item: unknown;
+        error: Error;
+    }>;
+    /** Summary */
+    summary: {
+        total: number;
+        successful: number;
+        failed: number;
+    };
+}
+/**
+ * Batch create items
+ *
+ * @example
+ * ```typescript
+ * const traces = [
+ *   { name: 'trace-1', traceId: 'id-1' },
+ *   { name: 'trace-2', traceId: 'id-2' },
+ *   // ... 1000 more
+ * ];
+ *
+ * const result = await batchCreate(
+ *   (item) => client.traces.create(item),
+ *   traces,
+ *   {
+ *     batchSize: 100,
+ *     onProgress: (p) => console.log(`${p.completed}/${p.total}`)
+ *   }
+ * );
+ * ```
+ */
+export declare function batchProcess<TInput, TOutput>(processor: (item: TInput) => Promise<TOutput>, items: TInput[], options?: BatchOptions): Promise<BatchResult<TOutput>>;
+/**
+ * Stream evaluation results
+ *
+ * @example
+ * ```typescript
+ * const config = {
+ *   cases: [...],
+ *   executor: async (input) => callLLM(input)
+ * };
+ *
+ * for await (const result of streamEvaluation(config)) {
+ *   console.log(`Case ${result.caseId}: ${result.passed ? 'PASS' : 'FAIL'}`);
+ *   console.log(`Progress: ${result.completed}/${result.total}`);
+ * }
+ * ```
+ */
+export declare function streamEvaluation<T>(config: {
+    cases: T[];
+    executor: (testCase: T) => Promise<unknown>;
+    onProgress?: (progress: BatchProgress) => void;
+}): AsyncGenerator<{
+    caseId: string;
+    case: T;
+    result: unknown;
+    passed: boolean;
+    completed: number;
+    total: number;
+}>;
+/**
+ * Batch read with pagination
+ *
+ * @example
+ * ```typescript
+ * const allTraces = await batchRead(
+ *   (params) => client.traces.list(params),
+ *   { pageSize: 100 }
+ * );
+ * ```
+ */
+export declare function batchRead<T>(fetcher: (params: {
+    limit: number;
+    offset: number;
+}) => Promise<T[]>, options?: {
+    pageSize?: number;
+    maxPages?: number;
+    onProgress?: (page: number, items: number) => void;
+}): Promise<T[]>;
+/**
+ * Rate-limited batch processor
+ *
+ * @example
+ * ```typescript
+ * const limiter = new RateLimiter({ requestsPerSecond: 10 });
+ *
+ * for (const item of items) {
+ *   await limiter.throttle(() => client.traces.create(item));
+ * }
+ * ```
+ */
+export declare class RateLimiter {
+    private queue;
+    private processing;
+    private requestsPerSecond;
+    private interval;
+    constructor(options: {
+        requestsPerSecond: number;
+    });
+    /**
+     * Throttle a function call
+     */
+    throttle<T>(fn: () => Promise<T>): Promise<T>;
+    private process;
+}
+/**
+ * Chunk array into smaller arrays
+ *
+ * @example
+ * ```typescript
+ * const chunks = chunk([1, 2, 3, 4, 5], 2);
+ * // [[1, 2], [3, 4], [5]]
+ * ```
+ */
+export declare function chunk<T>(array: T[], size: number): T[][];