@pauly4010/evalai-sdk 1.3.0

package/dist/batch.js

@@ -0,0 +1,178 @@
+"use strict";
+/**
+ * Request batching for improved performance
+ * Combines multiple API requests into fewer network calls
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RequestBatcher = void 0;
+exports.canBatch = canBatch;
+exports.batchProcess = batchProcess;
+/**
+ * Batch processor for API requests
+ */
+class RequestBatcher {
+    constructor(executeBatch, options = {}) {
+        this.executeBatch = executeBatch;
+        this.queue = [];
+        this.batchTimer = null;
+        this.requestCounter = 0;
+        this.maxBatchSize = options.maxBatchSize || 10;
+        this.batchDelay = options.batchDelay || 50; // 50ms
+    }
+    /**
+     * Add request to batch queue
+     */
+    async enqueue(method, endpoint, body, headers) {
+        return new Promise((resolve, reject) => {
+            const id = `req_${this.requestCounter++}_${Date.now()}`;
+            this.queue.push({
+                id,
+                resolve,
+                reject,
+                request: { id, method, endpoint, body, headers },
+            });
+            // Process immediately if batch is full
+            if (this.queue.length >= this.maxBatchSize) {
+                this.processBatch();
+            }
+            else {
+                // Otherwise schedule batch processing
+                this.scheduleBatch();
+            }
+        });
+    }
+    /**
+     * Schedule batch processing after delay
+     */
+    scheduleBatch() {
+        if (this.batchTimer) {
+            return;
+        }
+        this.batchTimer = setTimeout(() => {
+            this.processBatch();
+        }, this.batchDelay);
+    }
+    /**
+     * Process current batch
+     */
+    async processBatch() {
+        if (this.batchTimer) {
+            if (typeof this.batchTimer === 'number') {
+                clearTimeout(this.batchTimer);
+            }
+            else {
+                clearTimeout(this.batchTimer);
+            }
+            this.batchTimer = null;
+        }
+        if (this.queue.length === 0) {
+            return;
+        }
+        // Take items from queue
+        const batch = this.queue.splice(0, this.maxBatchSize);
+        const requests = batch.map(item => item.request);
+        try {
+            const responses = await this.executeBatch(requests);
+            // Match responses to requests and resolve/reject
+            for (const response of responses) {
+                const pendingRequest = batch.find(item => item.id === response.id);
+                if (pendingRequest) {
+                    if (response.status >= 200 && response.status < 300) {
+                        pendingRequest.resolve(response.data);
+                    }
+                    else {
+                        pendingRequest.reject(new Error(response.error || `Request failed with status ${response.status}`));
+                    }
+                }
+            }
+            // Handle any requests that didn't get a response
+            for (const item of batch) {
+                if (!responses.find(r => r.id === item.id)) {
+                    item.reject(new Error('No response received for request'));
+                }
+            }
+        }
+        catch (error) {
+            // Reject all requests in batch on error
+            for (const item of batch) {
+                item.reject(error);
+            }
+        }
+        // If there are more items in queue, schedule next batch
+        if (this.queue.length > 0) {
+            this.scheduleBatch();
+        }
+    }
+    /**
+     * Flush all pending requests immediately
+     */
+    async flush() {
+        while (this.queue.length > 0) {
+            await this.processBatch();
+        }
+    }
+    /**
+     * Clear queue without processing
+     */
+    clear() {
+        if (this.batchTimer) {
+            if (typeof this.batchTimer === 'number') {
+                clearTimeout(this.batchTimer);
+            }
+            else {
+                clearTimeout(this.batchTimer);
+            }
+            this.batchTimer = null;
+        }
+        // Reject all pending requests
+        for (const item of this.queue) {
+            item.reject(new Error('Batch queue cleared'));
+        }
+        this.queue = [];
+    }
+    /**
+     * Get queue statistics
+     */
+    getStats() {
+        return {
+            queueSize: this.queue.length,
+            maxBatchSize: this.maxBatchSize,
+        };
+    }
+}
+exports.RequestBatcher = RequestBatcher;
+/**
+ * @internal - Internal SDK logic, not part of public API
+ * Check if requests can be batched together
+ */
+function canBatch(method, endpoint) {
+    if (method !== 'GET') {
+        return false;
+    }
+    const batchableEndpoints = [
+        '/traces',
+        '/evaluations',
+        '/annotations',
+        '/results',
+    ];
+    return batchableEndpoints.some(pattern => endpoint.includes(pattern));
+}
+/**
+ * Batch multiple async operations with concurrency limit
+ */
+async function batchProcess(items, processor, concurrency = 5) {
+    const results = [];
+    const executing = [];
+    for (const item of items) {
+        const promise = processor(item).then(result => {
+            results.push(result);
+        });
+        executing.push(promise);
+        if (executing.length >= concurrency) {
+            await Promise.race(executing);
+            executing.splice(executing.findIndex(p => p === promise), 1);
+        }
+    }
+    await Promise.all(executing);
+    return results;
+}

package/dist/cache.d.ts

@@ -0,0 +1,65 @@
+/**
+ * Simple in-memory cache with TTL for SDK requests
+ * Reduces redundant API calls and improves performance
+ */
+export declare class RequestCache {
+    private cache;
+    private maxSize;
+    constructor(maxSize?: number);
+    /**
+     * Generate cache key from request parameters
+     */
+    private generateKey;
+    /**
+     * Check if cache entry is still valid
+     */
+    private isValid;
+    /**
+     * Get cached response if valid
+     */
+    get<T>(method: string, url: string, params?: any): T | null;
+    /**
+     * Store response in cache
+     */
+    set<T>(method: string, url: string, data: T, ttl: number, params?: any): void;
+    /**
+     * Invalidate specific cache entry
+     */
+    invalidate(method: string, url: string, params?: any): void;
+    /**
+     * Invalidate all cache entries matching a pattern
+     */
+    invalidatePattern(pattern: string): void;
+    /**
+     * Clear all cache entries
+     */
+    clear(): void;
+    /**
+     * Get cache statistics
+     */
+    getStats(): {
+        size: number;
+        maxSize: number;
+        hitRate?: number;
+    };
+}
+/**
+ * Default cache TTL values (in milliseconds)
+ * @internal - Used by SDK internally, exposed for advanced customization only
+ */
+export declare const CacheTTL: {
+    readonly SHORT: number;
+    readonly MEDIUM: number;
+    readonly LONG: number;
+    readonly HOUR: number;
+};
+/**
+ * @internal - Internal SDK logic, not part of public API
+ * Determine if a request should be cached based on method and endpoint
+ */
+export declare function shouldCache(method: string, endpoint: string): boolean;
+/**
+ * @internal - Internal SDK logic, not part of public API
+ * Get appropriate TTL for an endpoint
+ */
+export declare function getTTL(endpoint: string): number;

package/dist/cache.js

@@ -0,0 +1,135 @@
+"use strict";
+/**
+ * Simple in-memory cache with TTL for SDK requests
+ * Reduces redundant API calls and improves performance
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CacheTTL = exports.RequestCache = void 0;
+exports.shouldCache = shouldCache;
+exports.getTTL = getTTL;
+class RequestCache {
+    constructor(maxSize = 1000) {
+        this.cache = new Map();
+        this.maxSize = maxSize;
+    }
+    /**
+     * Generate cache key from request parameters
+     */
+    generateKey(method, url, params) {
+        const paramString = params ? JSON.stringify(params) : '';
+        return `${method}:${url}:${paramString}`;
+    }
+    /**
+     * Check if cache entry is still valid
+     */
+    isValid(entry) {
+        return Date.now() - entry.timestamp < entry.ttl;
+    }
+    /**
+     * Get cached response if valid
+     */
+    get(method, url, params) {
+        const key = this.generateKey(method, url, params);
+        const entry = this.cache.get(key);
+        if (!entry) {
+            return null;
+        }
+        if (!this.isValid(entry)) {
+            this.cache.delete(key);
+            return null;
+        }
+        return entry.data;
+    }
+    /**
+     * Store response in cache
+     */
+    set(method, url, data, ttl, params) {
+        // Enforce cache size limit (LRU-style)
+        if (this.cache.size >= this.maxSize) {
+            const firstKey = this.cache.keys().next().value;
+            if (firstKey) {
+                this.cache.delete(firstKey);
+            }
+        }
+        const key = this.generateKey(method, url, params);
+        this.cache.set(key, {
+            data,
+            timestamp: Date.now(),
+            ttl,
+        });
+    }
+    /**
+     * Invalidate specific cache entry
+     */
+    invalidate(method, url, params) {
+        const key = this.generateKey(method, url, params);
+        this.cache.delete(key);
+    }
+    /**
+     * Invalidate all cache entries matching a pattern
+     */
+    invalidatePattern(pattern) {
+        for (const key of this.cache.keys()) {
+            if (key.includes(pattern)) {
+                this.cache.delete(key);
+            }
+        }
+    }
+    /**
+     * Clear all cache entries
+     */
+    clear() {
+        this.cache.clear();
+    }
+    /**
+     * Get cache statistics
+     */
+    getStats() {
+        return {
+            size: this.cache.size,
+            maxSize: this.maxSize,
+        };
+    }
+}
+exports.RequestCache = RequestCache;
+/**
+ * Default cache TTL values (in milliseconds)
+ * @internal - Used by SDK internally, exposed for advanced customization only
+ */
+exports.CacheTTL = {
+    SHORT: 30 * 1000,
+    MEDIUM: 5 * 60 * 1000,
+    LONG: 30 * 60 * 1000,
+    HOUR: 60 * 60 * 1000,
+};
+/**
+ * @internal - Internal SDK logic, not part of public API
+ * Determine if a request should be cached based on method and endpoint
+ */
+function shouldCache(method, endpoint) {
+    if (method !== 'GET') {
+        return false;
+    }
+    const noCacheEndpoints = [
+        '/health',
+        '/usage',
+        '/deliveries',
+    ];
+    return !noCacheEndpoints.some(pattern => endpoint.includes(pattern));
+}
+/**
+ * @internal - Internal SDK logic, not part of public API
+ * Get appropriate TTL for an endpoint
+ */
+function getTTL(endpoint) {
+    if (endpoint.includes('/api-keys') || endpoint.includes('/webhooks')) {
+        return exports.CacheTTL.LONG;
+    }
+    if (endpoint.includes('/evaluations') || endpoint.includes('/configs')) {
+        return exports.CacheTTL.MEDIUM;
+    }
+    if (endpoint.includes('/traces') || endpoint.includes('/results')) {
+        return exports.CacheTTL.SHORT;
+    }
+    return exports.CacheTTL.MEDIUM;
+}

package/dist/cli/index.d.ts

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+/**
+ * CLI for AI Evaluation Platform SDK
+ * Tier 2.6: CLI for Everything
+ */
+export {};

package/dist/cli/index.js

@@ -0,0 +1,181 @@
+#!/usr/bin/env node
+"use strict";
+/**
+ * CLI for AI Evaluation Platform SDK
+ * Tier 2.6: CLI for Everything
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+const client_1 = require("../client");
+const export_1 = require("../export");
+const program = new commander_1.Command();
+program
+    .name('evalai')
+    .description('AI Evaluation Platform CLI')
+    .version('1.0.0');
+// Initialize project
+program
+    .command('init')
+    .description('Initialize a new evaluation project')
+    .option('-d, --dir <directory>', 'Project directory', '.')
+    .action(async (options) => {
+    const dir = path_1.default.resolve(options.dir);
+    console.log('🚀 Initializing EvalAI project...');
+    // Create directory structure
+    await promises_1.default.mkdir(path_1.default.join(dir, '.evalai'), { recursive: true });
+    await promises_1.default.mkdir(path_1.default.join(dir, '.evalai', 'snapshots'), { recursive: true });
+    await promises_1.default.mkdir(path_1.default.join(dir, 'evaluations'), { recursive: true });
+    // Create config file
+    const config = {
+        apiKey: process.env.EVALAI_API_KEY || '',
+        projectId: process.env.EVALAI_PROJECT_ID || '',
+        baseUrl: 'http://localhost:3000/api',
+        debug: false,
+        retry: {
+            maxAttempts: 3,
+            backoff: 'exponential'
+        }
+    };
+    await promises_1.default.writeFile(path_1.default.join(dir, 'evalai.config.json'), JSON.stringify(config, null, 2));
+    // Create example evaluation file
+    const exampleEval = `import { AIEvalClient, createTestSuite, expect } from '@pauly4010/evalai-sdk'
+
+const client = AIEvalClient.init()
+
+const suite = createTestSuite('example-evaluation', {
+  cases: [
+    {
+      input: 'What is 2+2?',
+      expected: '4',
+      name: 'simple-math'
+    },
+    {
+      input: 'Explain AI in simple terms',
+      expected: (output) => {
+        expect(output).toContainKeywords(['artificial', 'intelligence'])
+        expect(output).toHaveLength({ min: 50, max: 500 })
+        return true
+      },
+      name: 'ai-explanation'
+    }
+  ]
+})
+
+// Run the test suite
+suite.run().then(results => {
+  console.log('Test Results:', results)
+  console.log(\`Passed: \${results.passed}/\${results.total}\`)
+})
+`;
+    await promises_1.default.writeFile(path_1.default.join(dir, 'evaluations', 'example.ts'), exampleEval);
+    console.log('✅ Project initialized successfully!');
+    console.log('\nNext steps:');
+    console.log('1. Set your API key: export EVALAI_API_KEY=your-key');
+    console.log('2. Set your project ID: export EVALAI_PROJECT_ID=your-project');
+    console.log('3. Run evaluations: npx evalai eval:run');
+});
+// Run evaluations
+program
+    .command('eval:run')
+    .description('Run evaluation tests')
+    .option('-c, --config <path>', 'Config file path', './evalai.config.json')
+    .option('-f, --file <path>', 'Evaluation file to run')
+    .action(async (options) => {
+    console.log('🧪 Running evaluations...');
+    // Load config
+    const configPath = path_1.default.resolve(options.config);
+    let config;
+    try {
+        const configContent = await promises_1.default.readFile(configPath, 'utf-8');
+        config = JSON.parse(configContent);
+    }
+    catch (error) {
+        console.error('❌ Config file not found. Run "evalai init" first.');
+        process.exit(1);
+    }
+    const client = client_1.AIEvalClient.init(config);
+    // If file specified, run that file
+    if (options.file) {
+        console.log(`Running ${options.file}...`);
+        // Dynamic import of evaluation file would go here
+        // This requires compilation step for TS files
+    }
+    else {
+        // Run all evaluations in the evaluations directory
+        console.log('Running all evaluations...');
+    }
+    console.log('✅ Evaluations completed!');
+});
+// List traces
+program
+    .command('traces')
+    .description('List and filter traces')
+    .option('-l, --limit <number>', 'Number of traces to show', '10')
+    .option('--failed', 'Show only failed traces')
+    .option('--slow', 'Show slow traces (>5s)')
+    .action(async (options) => {
+    const configPath = path_1.default.resolve('./evalai.config.json');
+    let config;
+    try {
+        const configContent = await promises_1.default.readFile(configPath, 'utf-8');
+        config = JSON.parse(configContent);
+    }
+    catch (error) {
+        console.error('❌ Config file not found. Run "evalai init" first.');
+        process.exit(1);
+    }
+    const client = client_1.AIEvalClient.init(config);
+    console.log('📊 Fetching traces...');
+    // API call to get traces would go here
+    console.log(`Showing ${options.limit} traces`);
+});
+// Export data
+program
+    .command('export')
+    .description('Export data from EvalAI')
+    .option('-f, --format <format>', 'Export format (json, csv, xlsx)', 'json')
+    .option('-o, --output <path>', 'Output file path', './export')
+    .option('-t, --type <type>', 'Data type (traces, evaluations, all)', 'all')
+    .action(async (options) => {
+    const configPath = path_1.default.resolve('./evalai.config.json');
+    let config;
+    try {
+        const configContent = await promises_1.default.readFile(configPath, 'utf-8');
+        config = JSON.parse(configContent);
+    }
+    catch (error) {
+        console.error('❌ Config file not found. Run "evalai init" first.');
+        process.exit(1);
+    }
+    const client = client_1.AIEvalClient.init(config);
+    console.log(`📥 Exporting data as ${options.format}...`);
+    const data = await (0, export_1.exportData)(client, {
+        format: options.format,
+        includeTraces: true,
+        includeEvaluations: true
+    });
+    // Save to file
+    const outputPath = path_1.default.resolve(process.cwd(), options.output);
+    await promises_1.default.writeFile(outputPath, JSON.stringify(data, null, 2));
+    console.log(`✅ Data exported to ${outputPath}`);
+});
+// Dev server
+program
+    .command('dev')
+    .description('Start local development server')
+    .option('-p, --port <port>', 'Port number', '3001')
+    .action(async (options) => {
+    console.log(`🚀 Starting development server on port ${options.port}...`);
+    console.log('📊 Dashboard: http://localhost:' + options.port);
+    console.log('🔍 API: http://localhost:' + options.port + '/api');
+    console.log('\nPress Ctrl+C to stop');
+    // This would start an Express server with a simple dashboard
+    // For now, just keep the process running
+    process.stdin.resume();
+});
+program.parse();