@pauly4010/evalai-sdk 1.3.0

package/dist/streaming.js

@@ -0,0 +1,268 @@
+"use strict";
+/**
+ * Streaming & Batch Operations
+ * Tier 2.8: Handle large datasets efficiently
+ *
+ * @example
+ * ```typescript
+ * import { streamEvaluations, batchCreate } from '@ai-eval-platform/sdk';
+ *
+ * // Stream large evaluation results
+ * for await (const result of streamEvaluations(client, config)) {
+ *   console.log(`Progress: ${result.completed}/${result.total}`);
+ * }
+ *
+ * // Batch create traces
+ * await batchCreate(client.traces, traces, { batchSize: 100 });
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RateLimiter = void 0;
+exports.batchProcess = batchProcess;
+exports.streamEvaluation = streamEvaluation;
+exports.batchRead = batchRead;
+exports.chunk = chunk;
+/**
+ * Batch create items
+ *
+ * @example
+ * ```typescript
+ * const traces = [
+ *   { name: 'trace-1', traceId: 'id-1' },
+ *   { name: 'trace-2', traceId: 'id-2' },
+ *   // ... 1000 more
+ * ];
+ *
+ * const result = await batchCreate(
+ *   (item) => client.traces.create(item),
+ *   traces,
+ *   {
+ *     batchSize: 100,
+ *     onProgress: (p) => console.log(`${p.completed}/${p.total}`)
+ *   }
+ * );
+ * ```
+ */
+async function batchProcess(processor, items, options = {}) {
+    const { batchSize = 100, parallel = true, delayMs = 0, onProgress, onError, continueOnError = true } = options;
+    const result = {
+        successful: [],
+        failed: [],
+        summary: {
+            total: items.length,
+            successful: 0,
+            failed: 0
+        }
+    };
+    // Split into batches
+    const batches = [];
+    for (let i = 0; i < items.length; i += batchSize) {
+        batches.push(items.slice(i, i + batchSize));
+    }
+    // Process batches
+    for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
+        const batch = batches[batchIndex];
+        const processBatch = async () => {
+            const batchPromises = batch.map(async (item, itemIndex) => {
+                try {
+                    const output = await processor(item);
+                    result.successful.push(output);
+                    result.summary.successful++;
+                    return { success: true, output };
+                }
+                catch (error) {
+                    const batchError = {
+                        batch: batchIndex,
+                        index: itemIndex,
+                        error: error instanceof Error ? error : new Error(String(error)),
+                        item
+                    };
+                    result.failed.push({
+                        item,
+                        error: batchError.error
+                    });
+                    result.summary.failed++;
+                    if (onError)
+                        onError(batchError);
+                    if (!continueOnError) {
+                        throw error;
+                    }
+                    return { success: false, error };
+                }
+            });
+            if (parallel) {
+                await Promise.all(batchPromises);
+            }
+            else {
+                for (const promise of batchPromises) {
+                    await promise;
+                }
+            }
+        };
+        await processBatch();
+        // Progress callback
+        if (onProgress) {
+            onProgress({
+                total: items.length,
+                completed: result.summary.successful + result.summary.failed,
+                failed: result.summary.failed,
+                batch: batchIndex + 1,
+                totalBatches: batches.length
+            });
+        }
+        // Delay between batches
+        if (delayMs > 0 && batchIndex < batches.length - 1) {
+            await new Promise(resolve => setTimeout(resolve, delayMs));
+        }
+    }
+    return result;
+}
+/**
+ * Stream evaluation results
+ *
+ * @example
+ * ```typescript
+ * const config = {
+ *   cases: [...],
+ *   executor: async (input) => callLLM(input)
+ * };
+ *
+ * for await (const result of streamEvaluation(config)) {
+ *   console.log(`Case ${result.caseId}: ${result.passed ? 'PASS' : 'FAIL'}`);
+ *   console.log(`Progress: ${result.completed}/${result.total}`);
+ * }
+ * ```
+ */
+async function* streamEvaluation(config) {
+    const { cases, executor } = config;
+    let completed = 0;
+    for (const [index, testCase] of cases.entries()) {
+        try {
+            const result = await executor(testCase);
+            completed++;
+            yield {
+                caseId: `case-${index}`,
+                case: testCase,
+                result,
+                passed: true,
+                completed,
+                total: cases.length
+            };
+        }
+        catch (error) {
+            completed++;
+            yield {
+                caseId: `case-${index}`,
+                case: testCase,
+                result: error,
+                passed: false,
+                completed,
+                total: cases.length
+            };
+        }
+    }
+}
+/**
+ * Batch read with pagination
+ *
+ * @example
+ * ```typescript
+ * const allTraces = await batchRead(
+ *   (params) => client.traces.list(params),
+ *   { pageSize: 100 }
+ * );
+ * ```
+ */
+async function batchRead(fetcher, options = {}) {
+    const { pageSize = 100, maxPages, onProgress } = options;
+    const allItems = [];
+    let page = 0;
+    let hasMore = true;
+    while (hasMore && (!maxPages || page < maxPages)) {
+        const items = await fetcher({
+            limit: pageSize,
+            offset: page * pageSize
+        });
+        if (items.length === 0) {
+            hasMore = false;
+        }
+        else {
+            allItems.push(...items);
+            page++;
+            if (onProgress) {
+                onProgress(page, allItems.length);
+            }
+            if (items.length < pageSize) {
+                hasMore = false;
+            }
+        }
+    }
+    return allItems;
+}
+/**
+ * Rate-limited batch processor
+ *
+ * @example
+ * ```typescript
+ * const limiter = new RateLimiter({ requestsPerSecond: 10 });
+ *
+ * for (const item of items) {
+ *   await limiter.throttle(() => client.traces.create(item));
+ * }
+ * ```
+ */
+class RateLimiter {
+    constructor(options) {
+        this.queue = [];
+        this.processing = false;
+        this.requestsPerSecond = options.requestsPerSecond;
+        this.interval = 1000 / options.requestsPerSecond;
+    }
+    /**
+     * Throttle a function call
+     */
+    async throttle(fn) {
+        return new Promise((resolve, reject) => {
+            this.queue.push(async () => {
+                try {
+                    const result = await fn();
+                    resolve(result);
+                }
+                catch (error) {
+                    reject(error);
+                }
+            });
+            if (!this.processing) {
+                this.process();
+            }
+        });
+    }
+    async process() {
+        this.processing = true;
+        while (this.queue.length > 0) {
+            const fn = this.queue.shift();
+            if (fn) {
+                await fn();
+                await new Promise(resolve => setTimeout(resolve, this.interval));
+            }
+        }
+        this.processing = false;
+    }
+}
+exports.RateLimiter = RateLimiter;
+/**
+ * Chunk array into smaller arrays
+ *
+ * @example
+ * ```typescript
+ * const chunks = chunk([1, 2, 3, 4, 5], 2);
+ * // [[1, 2], [3, 4], [5]]
+ * ```
+ */
+function chunk(array, size) {
+    const chunks = [];
+    for (let i = 0; i < array.length; i += size) {
+        chunks.push(array.slice(i, i + size));
+    }
+    return chunks;
+}

package/dist/testing.d.ts

@@ -0,0 +1,204 @@
+/**
+ * Test Suite Builder
+ * Tier 2.7: Declarative test definitions
+ *
+ * @example
+ * ```typescript
+ * import { createTestSuite, expect } from '@ai-eval-platform/sdk';
+ *
+ * const suite = createTestSuite('chatbot-responses', {
+ *   cases: [
+ *     {
+ *       input: 'Hello',
+ *       assertions: [
+ *         (output) => expect(output).toContain('greeting'),
+ *         (output) => expect(output).toHaveSentiment('positive')
+ *       ]
+ *     }
+ *   ]
+ * });
+ *
+ * const results = await suite.run();
+ * ```
+ */
+import { AssertionResult } from './assertions';
+/**
+ * Test suite case definition (different from API TestCase type)
+ * Use this for defining test cases in test suites with assertions
+ */
+export interface TestSuiteCase {
+    /** Unique identifier for the test case */
+    id?: string;
+    /** Input to the LLM */
+    input: string;
+    /** Expected output (optional) */
+    expected?: string;
+    /** Metadata for the test case */
+    metadata?: Record<string, any>;
+    /** Assertion functions to run */
+    assertions?: ((output: string) => AssertionResult)[];
+}
+/** @deprecated Use TestSuiteCase instead to avoid confusion with API TestCase type */
+export type TestCase = TestSuiteCase;
+export interface TestSuiteConfig {
+    /** Test cases to run */
+    cases: TestSuiteCase[];
+    /** Function that generates output from input */
+    executor?: (input: string) => Promise<string>;
+    /** Run tests in parallel (default: true) */
+    parallel?: boolean;
+    /** Stop on first failure (default: false) */
+    stopOnFailure?: boolean;
+    /** Timeout per test case in ms (default: 30000) */
+    timeout?: number;
+}
+export interface TestSuiteCaseResult {
+    /** Test case ID */
+    id: string;
+    /** Input that was tested */
+    input: string;
+    /** Expected output */
+    expected?: string;
+    /** Actual output */
+    actual: string;
+    /** Whether test passed */
+    passed: boolean;
+    /** Assertion results */
+    assertions: AssertionResult[];
+    /** Duration in milliseconds */
+    durationMs: number;
+    /** Error if test failed to execute */
+    error?: string;
+}
+/** @deprecated Use TestSuiteCaseResult instead */
+export type TestCaseResult = TestSuiteCaseResult;
+export interface TestSuiteResult {
+    /** Suite name */
+    name: string;
+    /** Total number of test cases */
+    total: number;
+    /** Number of passed tests */
+    passed: number;
+    /** Number of failed tests */
+    failed: number;
+    /** Total duration in milliseconds */
+    durationMs: number;
+    /** Individual test results */
+    results: TestSuiteCaseResult[];
+}
+/**
+ * Test Suite for declarative evaluation testing
+ */
+export declare class TestSuite {
+    private name;
+    private config;
+    constructor(name: string, config: TestSuiteConfig);
+    /**
+     * Run all test cases
+     *
+     * @example
+     * ```typescript
+     * const results = await suite.run();
+     * console.log(`${results.passed}/${results.total} tests passed`);
+     * ```
+     */
+    run(): Promise<TestSuiteResult>;
+    /**
+     * Add a test case to the suite
+     */
+    addCase(testCase: TestSuiteCase): void;
+    /**
+     * Get suite configuration
+     */
+    getConfig(): TestSuiteConfig;
+}
+/**
+ * Create a test suite
+ *
+ * @example
+ * ```typescript
+ * const suite = createTestSuite('my-tests', {
+ *   cases: [
+ *     {
+ *       input: 'Hello',
+ *       assertions: [
+ *         (output) => expect(output).toContain('hi'),
+ *         (output) => expect(output).toHaveSentiment('positive')
+ *       ]
+ *     }
+ *   ],
+ *   executor: async (input) => {
+ *     // Your LLM call here
+ *     return callLLM(input);
+ *   }
+ * });
+ * ```
+ */
+export declare function createTestSuite(name: string, config: TestSuiteConfig): TestSuite;
+/**
+ * Helper to create assertions from expected keywords
+ *
+ * @example
+ * ```typescript
+ * const suite = createTestSuite('tests', {
+ *   cases: [
+ *     {
+ *       input: 'refund policy',
+ *       assertions: containsKeywords(['refund', 'return', 'policy'])
+ *     }
+ *   ]
+ * });
+ * ```
+ */
+export declare function containsKeywords(keywords: string[]): (output: string) => AssertionResult;
+/**
+ * Helper to create pattern matching assertion
+ *
+ * @example
+ * ```typescript
+ * const suite = createTestSuite('tests', {
+ *   cases: [
+ *     {
+ *       input: 'What time is it?',
+ *       assertions: matchesPattern(/\d{1,2}:\d{2}/)
+ *     }
+ *   ]
+ * });
+ * ```
+ */
+export declare function matchesPattern(pattern: RegExp): (output: string) => AssertionResult;
+/**
+ * Helper to create sentiment assertion
+ *
+ * @example
+ * ```typescript
+ * const suite = createTestSuite('tests', {
+ *   cases: [
+ *     {
+ *       input: 'Thank you!',
+ *       assertions: hasSentiment('positive')
+ *     }
+ *   ]
+ * });
+ * ```
+ */
+export declare function hasSentiment(sentiment: 'positive' | 'negative' | 'neutral'): (output: string) => AssertionResult;
+/**
+ * Helper to create length range assertion
+ *
+ * @example
+ * ```typescript
+ * const suite = createTestSuite('tests', {
+ *   cases: [
+ *     {
+ *       input: 'Summarize this',
+ *       assertions: hasLength({ min: 50, max: 200 })
+ *     }
+ *   ]
+ * });
+ * ```
+ */
+export declare function hasLength(range: {
+    min?: number;
+    max?: number;
+}): (output: string) => AssertionResult;

package/dist/testing.js

@@ -0,0 +1,252 @@
+"use strict";
+/**
+ * Test Suite Builder
+ * Tier 2.7: Declarative test definitions
+ *
+ * @example
+ * ```typescript
+ * import { createTestSuite, expect } from '@ai-eval-platform/sdk';
+ *
+ * const suite = createTestSuite('chatbot-responses', {
+ *   cases: [
+ *     {
+ *       input: 'Hello',
+ *       assertions: [
+ *         (output) => expect(output).toContain('greeting'),
+ *         (output) => expect(output).toHaveSentiment('positive')
+ *       ]
+ *     }
+ *   ]
+ * });
+ *
+ * const results = await suite.run();
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TestSuite = void 0;
+exports.createTestSuite = createTestSuite;
+exports.containsKeywords = containsKeywords;
+exports.matchesPattern = matchesPattern;
+exports.hasSentiment = hasSentiment;
+exports.hasLength = hasLength;
+const assertions_1 = require("./assertions");
+/**
+ * Test Suite for declarative evaluation testing
+ */
+class TestSuite {
+    constructor(name, config) {
+        this.name = name;
+        this.config = config;
+    }
+    /**
+     * Run all test cases
+     *
+     * @example
+     * ```typescript
+     * const results = await suite.run();
+     * console.log(`${results.passed}/${results.total} tests passed`);
+     * ```
+     */
+    async run() {
+        const startTime = Date.now();
+        const results = [];
+        const runTestCase = async (testCase, index) => {
+            const caseStartTime = Date.now();
+            const id = testCase.id || `case-${index}`;
+            try {
+                // Execute to get output
+                let actual;
+                if (this.config.executor) {
+                    const timeout = this.config.timeout || 30000;
+                    const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error(`Test timeout after ${timeout}ms`)), timeout));
+                    actual = await Promise.race([
+                        this.config.executor(testCase.input),
+                        timeoutPromise
+                    ]);
+                }
+                else if (testCase.expected) {
+                    actual = testCase.expected; // Use expected as actual if no executor
+                }
+                else {
+                    throw new Error('No executor provided and no expected output');
+                }
+                // Run assertions
+                const assertions = [];
+                let allPassed = true;
+                // Run custom assertions
+                if (testCase.assertions) {
+                    for (const assertion of testCase.assertions) {
+                        const result = assertion(actual);
+                        assertions.push(result);
+                        if (!result.passed)
+                            allPassed = false;
+                    }
+                }
+                // Default equality check if expected provided
+                if (testCase.expected && !testCase.assertions) {
+                    const result = (0, assertions_1.expect)(actual).toEqual(testCase.expected);
+                    assertions.push(result);
+                    if (!result.passed)
+                        allPassed = false;
+                }
+                const durationMs = Date.now() - caseStartTime;
+                return {
+                    id,
+                    input: testCase.input,
+                    expected: testCase.expected,
+                    actual,
+                    passed: allPassed,
+                    assertions,
+                    durationMs
+                };
+            }
+            catch (error) {
+                const durationMs = Date.now() - caseStartTime;
+                return {
+                    id,
+                    input: testCase.input,
+                    expected: testCase.expected,
+                    actual: '',
+                    passed: false,
+                    assertions: [],
+                    durationMs,
+                    error: error instanceof Error ? error.message : String(error)
+                };
+            }
+        };
+        // Run tests
+        if (this.config.parallel) {
+            results.push(...await Promise.all(this.config.cases.map((tc, i) => runTestCase(tc, i))));
+        }
+        else {
+            for (let i = 0; i < this.config.cases.length; i++) {
+                const result = await runTestCase(this.config.cases[i], i);
+                results.push(result);
+                if (this.config.stopOnFailure && !result.passed) {
+                    break;
+                }
+            }
+        }
+        const durationMs = Date.now() - startTime;
+        const passed = results.filter(r => r.passed).length;
+        const failed = results.filter(r => !r.passed).length;
+        return {
+            name: this.name,
+            total: results.length,
+            passed,
+            failed,
+            durationMs,
+            results
+        };
+    }
+    /**
+     * Add a test case to the suite
+     */
+    addCase(testCase) {
+        this.config.cases.push(testCase);
+    }
+    /**
+     * Get suite configuration
+     */
+    getConfig() {
+        return { ...this.config };
+    }
+}
+exports.TestSuite = TestSuite;
+/**
+ * Create a test suite
+ *
+ * @example
+ * ```typescript
+ * const suite = createTestSuite('my-tests', {
+ *   cases: [
+ *     {
+ *       input: 'Hello',
+ *       assertions: [
+ *         (output) => expect(output).toContain('hi'),
+ *         (output) => expect(output).toHaveSentiment('positive')
+ *       ]
+ *     }
+ *   ],
+ *   executor: async (input) => {
+ *     // Your LLM call here
+ *     return callLLM(input);
+ *   }
+ * });
+ * ```
+ */
+function createTestSuite(name, config) {
+    return new TestSuite(name, config);
+}
+/**
+ * Helper to create assertions from expected keywords
+ *
+ * @example
+ * ```typescript
+ * const suite = createTestSuite('tests', {
+ *   cases: [
+ *     {
+ *       input: 'refund policy',
+ *       assertions: containsKeywords(['refund', 'return', 'policy'])
+ *     }
+ *   ]
+ * });
+ * ```
+ */
+function containsKeywords(keywords) {
+    return (output) => (0, assertions_1.expect)(output).toContainKeywords(keywords);
+}
+/**
+ * Helper to create pattern matching assertion
+ *
+ * @example
+ * ```typescript
+ * const suite = createTestSuite('tests', {
+ *   cases: [
+ *     {
+ *       input: 'What time is it?',
+ *       assertions: matchesPattern(/\d{1,2}:\d{2}/)
+ *     }
+ *   ]
+ * });
+ * ```
+ */
+function matchesPattern(pattern) {
+    return (output) => (0, assertions_1.expect)(output).toMatchPattern(pattern);
+}
+/**
+ * Helper to create sentiment assertion
+ *
+ * @example
+ * ```typescript
+ * const suite = createTestSuite('tests', {
+ *   cases: [
+ *     {
+ *       input: 'Thank you!',
+ *       assertions: hasSentiment('positive')
+ *     }
+ *   ]
+ * });
+ * ```
+ */
+function hasSentiment(sentiment) {
+    return (output) => (0, assertions_1.expect)(output).toHaveSentiment(sentiment);
+}
+/**
+ * Helper to create length range assertion
+ *
+ * @example
+ * ```typescript
+ * const suite = createTestSuite('tests', {
+ *   cases: [
+ *     {
+ *       input: 'Summarize this',
+ *       assertions: hasLength({ min: 50, max: 200 })
+ *     }
+ *   ]
+ * });
+ * ```
+ */
+function hasLength(range) {
+    return (output) => (0, assertions_1.expect)(output).toHaveLength(range);
+}