@pauly4010/evalai-sdk 1.4.1 → 1.5.0

package/dist/__tests__/testing.test.js

@@ -1,230 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const vitest_1 = require("vitest");
-const testing_1 = require("../testing");
-const assertions_1 = require("../assertions");
-(0, vitest_1.describe)('TestSuite', () => {
-    (0, vitest_1.describe)('basic execution', () => {
-        (0, vitest_1.it)('should run test cases with an executor', async () => {
-            const suite = (0, testing_1.createTestSuite)('basic-tests', {
-                cases: [
-                    {
-                        input: 'Hello',
-                        assertions: [
-                            (output) => (0, assertions_1.expect)(output).toContain('Hello'),
-                        ],
-                    },
-                ],
-                executor: async (input) => `Echo: ${input}`,
-            });
-            const result = await suite.run();
-            (0, vitest_1.expect)(result.name).toBe('basic-tests');
-            (0, vitest_1.expect)(result.total).toBe(1);
-            (0, vitest_1.expect)(result.passed).toBe(1);
-            (0, vitest_1.expect)(result.failed).toBe(0);
-            (0, vitest_1.expect)(result.results[0].passed).toBe(true);
-        });
-        (0, vitest_1.it)('should fail when assertion fails', async () => {
-            const suite = (0, testing_1.createTestSuite)('fail-tests', {
-                cases: [
-                    {
-                        input: 'Hello',
-                        assertions: [
-                            (output) => (0, assertions_1.expect)(output).toContain('missing keyword'),
-                        ],
-                    },
-                ],
-                executor: async (input) => `Echo: ${input}`,
-            });
-            const result = await suite.run();
-            (0, vitest_1.expect)(result.passed).toBe(0);
-            (0, vitest_1.expect)(result.failed).toBe(1);
-            (0, vitest_1.expect)(result.results[0].passed).toBe(false);
-        });
-    });
-    (0, vitest_1.describe)('default equality check', () => {
-        (0, vitest_1.it)('should use toEqual when expected is provided without assertions', async () => {
-            const suite = (0, testing_1.createTestSuite)('equality-tests', {
-                cases: [
-                    { input: 'hello', expected: 'hello' },
-                ],
-                // No executor — uses expected as actual
-            });
-            const result = await suite.run();
-            (0, vitest_1.expect)(result.passed).toBe(1);
-        });
-        (0, vitest_1.it)('should fail when expected does not match', async () => {
-            const suite = (0, testing_1.createTestSuite)('equality-fail', {
-                cases: [
-                    { input: 'hello', expected: 'world' },
-                ],
-                executor: async (input) => input, // Returns 'hello', not 'world'
-            });
-            const result = await suite.run();
-            (0, vitest_1.expect)(result.failed).toBe(1);
-        });
-    });
-    (0, vitest_1.describe)('parallel execution', () => {
-        (0, vitest_1.it)('should run tests in parallel', async () => {
-            const order = [];
-            const suite = (0, testing_1.createTestSuite)('parallel-tests', {
-                cases: [
-                    { id: '1', input: 'a', expected: 'a' },
-                    { id: '2', input: 'b', expected: 'b' },
-                    { id: '3', input: 'c', expected: 'c' },
-                ],
-                executor: async (input) => {
-                    order.push(parseInt(input, 36) - 9); // a=1, b=2, c=3
-                    return input;
-                },
-                parallel: true,
-            });
-            const result = await suite.run();
-            (0, vitest_1.expect)(result.total).toBe(3);
-            (0, vitest_1.expect)(result.passed).toBe(3);
-        });
-    });
-    (0, vitest_1.describe)('sequential execution', () => {
-        (0, vitest_1.it)('should run tests sequentially', async () => {
-            const order = [];
-            const suite = (0, testing_1.createTestSuite)('sequential-tests', {
-                cases: [
-                    { id: 'first', input: 'a', expected: 'a' },
-                    { id: 'second', input: 'b', expected: 'b' },
-                ],
-                executor: async (input) => {
-                    order.push(input);
-                    return input;
-                },
-                parallel: false,
-            });
-            const result = await suite.run();
-            (0, vitest_1.expect)(order).toEqual(['a', 'b']);
-            (0, vitest_1.expect)(result.passed).toBe(2);
-        });
-    });
-    (0, vitest_1.describe)('stopOnFailure', () => {
-        (0, vitest_1.it)('should stop after first failure when enabled', async () => {
-            const suite = (0, testing_1.createTestSuite)('stop-on-fail', {
-                cases: [
-                    { id: 'pass', input: 'hello', expected: 'hello' },
-                    { id: 'fail', input: 'hello', expected: 'nope' },
-                    { id: 'skip', input: 'hello', expected: 'hello' },
-                ],
-                executor: async (input) => input,
-                parallel: false,
-                stopOnFailure: true,
-            });
-            const result = await suite.run();
-            (0, vitest_1.expect)(result.total).toBe(2); // Only 2 ran
-            (0, vitest_1.expect)(result.passed).toBe(1);
-            (0, vitest_1.expect)(result.failed).toBe(1);
-        });
-    });
-    (0, vitest_1.describe)('timeout', () => {
-        (0, vitest_1.beforeEach)(() => {
-            vitest_1.vi.useFakeTimers();
-        });
-        (0, vitest_1.afterEach)(() => {
-            vitest_1.vi.useRealTimers();
-        });
-        (0, vitest_1.it)('should timeout slow tests', async () => {
-            const suite = (0, testing_1.createTestSuite)('timeout-tests', {
-                cases: [
-                    { id: 'slow', input: 'hello' },
-                ],
-                executor: async (_input) => {
-                    return new Promise((resolve) => {
-                        setTimeout(() => resolve('done'), 60000);
-                    });
-                },
-                timeout: 100,
-                parallel: false,
-            });
-            const runPromise = suite.run();
-            // Advance timers past the timeout
-            vitest_1.vi.advanceTimersByTime(200);
-            const result = await runPromise;
-            (0, vitest_1.expect)(result.results[0].passed).toBe(false);
-            (0, vitest_1.expect)(result.results[0].error).toContain('timeout');
-        });
-    });
-    (0, vitest_1.describe)('error handling', () => {
-        (0, vitest_1.it)('should catch executor errors gracefully', async () => {
-            const suite = (0, testing_1.createTestSuite)('error-tests', {
-                cases: [
-                    { input: 'hello' },
-                ],
-                executor: async () => {
-                    throw new Error('executor broke');
-                },
-            });
-            const result = await suite.run();
-            (0, vitest_1.expect)(result.results[0].passed).toBe(false);
-            (0, vitest_1.expect)(result.results[0].error).toBe('executor broke');
-        });
-        (0, vitest_1.it)('should fail when no executor and no expected', async () => {
-            const suite = (0, testing_1.createTestSuite)('no-exec', {
-                cases: [{ input: 'hello' }],
-            });
-            const result = await suite.run();
-            (0, vitest_1.expect)(result.results[0].passed).toBe(false);
-            (0, vitest_1.expect)(result.results[0].error).toContain('No executor');
-        });
-    });
-    (0, vitest_1.describe)('addCase', () => {
-        (0, vitest_1.it)('should allow adding cases after construction', async () => {
-            const suite = (0, testing_1.createTestSuite)('dynamic', {
-                cases: [],
-                executor: async (input) => input,
-            });
-            suite.addCase({ input: 'test', expected: 'test' });
-            const result = await suite.run();
-            (0, vitest_1.expect)(result.total).toBe(1);
-            (0, vitest_1.expect)(result.passed).toBe(1);
-        });
-    });
-    (0, vitest_1.describe)('custom assertion IDs', () => {
-        (0, vitest_1.it)('should use provided IDs', async () => {
-            const suite = (0, testing_1.createTestSuite)('ids', {
-                cases: [
-                    { id: 'custom-id', input: 'test', expected: 'test' },
-                ],
-            });
-            const result = await suite.run();
-            (0, vitest_1.expect)(result.results[0].id).toBe('custom-id');
-        });
-        (0, vitest_1.it)('should generate IDs when not provided', async () => {
-            const suite = (0, testing_1.createTestSuite)('auto-ids', {
-                cases: [
-                    { input: 'test', expected: 'test' },
-                ],
-            });
-            const result = await suite.run();
-            (0, vitest_1.expect)(result.results[0].id).toBe('case-0');
-        });
-    });
-});
-(0, vitest_1.describe)('Testing helper functions', () => {
-    (0, vitest_1.it)('containsKeywords returns an assertion function', () => {
-        const assertFn = (0, testing_1.containsKeywords)(['hello', 'world']);
-        const result = assertFn('hello world');
-        (0, vitest_1.expect)(result.passed).toBe(true);
-        (0, vitest_1.expect)(result.name).toBe('toContainKeywords');
-    });
-    (0, vitest_1.it)('matchesPattern returns an assertion function', () => {
-        const assertFn = (0, testing_1.matchesPattern)(/\d{3}/);
-        const result = assertFn('code: 123');
-        (0, vitest_1.expect)(result.passed).toBe(true);
-    });
-    (0, vitest_1.it)('hasSentiment returns an assertion function', () => {
-        const assertFn = (0, testing_1.hasSentiment)('positive');
-        const result = assertFn('This is great!');
-        (0, vitest_1.expect)(result.passed).toBe(true);
-    });
-    (0, vitest_1.it)('hasLength returns an assertion function', () => {
-        const assertFn = (0, testing_1.hasLength)({ min: 5, max: 50 });
-        const result = assertFn('hello world');
-        (0, vitest_1.expect)(result.passed).toBe(true);
-    });
-});

package/dist/__tests__/workflows.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/__tests__/workflows.test.js

@@ -1,222 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const vitest_1 = require("vitest");
-const workflows_1 = require("../workflows");
-const client_1 = require("../client");
-// Mock fetch
-const mockFetch = vitest_1.vi.fn();
-function createMockClient() {
-    vitest_1.vi.stubGlobal('fetch', mockFetch);
-    mockFetch.mockResolvedValue({
-        ok: true,
-        json: async () => ({ id: 1, traceId: 'trace-1', name: 'test' }),
-        status: 200,
-    });
-    return new client_1.AIEvalClient({ apiKey: 'test-key', baseUrl: 'http://localhost:3000', organizationId: 1 });
-}
-(0, vitest_1.describe)('WorkflowTracer', () => {
-    let client;
-    let tracer;
-    (0, vitest_1.beforeEach)(() => {
-        client = createMockClient();
-        tracer = new workflows_1.WorkflowTracer(client, { organizationId: 1 });
-    });
-    (0, vitest_1.afterEach)(() => {
-        vitest_1.vi.unstubAllGlobals();
-        mockFetch.mockReset();
-    });
-    (0, vitest_1.describe)('workflow lifecycle', () => {
-        (0, vitest_1.it)('should start and end a workflow', async () => {
-            const workflow = await tracer.startWorkflow('Test Pipeline');
-            (0, vitest_1.expect)(workflow.name).toBe('Test Pipeline');
-            (0, vitest_1.expect)(tracer.isWorkflowActive()).toBe(true);
-            await tracer.endWorkflow({ status: 'success' });
-            (0, vitest_1.expect)(tracer.isWorkflowActive()).toBe(false);
-        });
-        (0, vitest_1.it)('should throw if starting a second workflow', async () => {
-            await tracer.startWorkflow('First');
-            await (0, vitest_1.expect)(tracer.startWorkflow('Second')).rejects.toThrow('already active');
-        });
-        (0, vitest_1.it)('should throw if ending without starting', async () => {
-            await (0, vitest_1.expect)(tracer.endWorkflow()).rejects.toThrow('No active workflow');
-        });
-    });
-    (0, vitest_1.describe)('agent spans', () => {
-        (0, vitest_1.it)('should start and end agent spans', async () => {
-            await tracer.startWorkflow('Pipeline');
-            const span = await tracer.startAgentSpan('RouterAgent', { input: 'hello' });
-            (0, vitest_1.expect)(span.agentName).toBe('RouterAgent');
-            (0, vitest_1.expect)(span.spanId).toBeTruthy();
-            await tracer.endAgentSpan(span, { result: 'routed' });
-            await tracer.endWorkflow();
-        });
-        (0, vitest_1.it)('should throw if no workflow is active', async () => {
-            await (0, vitest_1.expect)(tracer.startAgentSpan('Agent')).rejects.toThrow('No active workflow');
-        });
-    });
-    (0, vitest_1.describe)('decision recording', () => {
-        (0, vitest_1.it)('should record decisions', async () => {
-            await tracer.startWorkflow('Pipeline');
-            await tracer.recordDecision({
-                agent: 'Router',
-                type: 'route',
-                chosen: 'technical_support',
-                alternatives: [{ action: 'billing', confidence: 30 }],
-                reasoning: 'Detected technical keywords',
-                confidence: 85,
-            });
-            const decisions = tracer.getDecisions();
-            (0, vitest_1.expect)(decisions).toHaveLength(1);
-            (0, vitest_1.expect)(decisions[0].agent).toBe('Router');
-            (0, vitest_1.expect)(decisions[0].chosen).toBe('technical_support');
-            (0, vitest_1.expect)(decisions[0].confidence).toBe(85);
-            await tracer.endWorkflow();
-        });
-        (0, vitest_1.it)('should throw if no workflow is active', async () => {
-            await (0, vitest_1.expect)(tracer.recordDecision({
-                agent: 'Agent',
-                type: 'action',
-                chosen: 'x',
-                alternatives: [],
-            })).rejects.toThrow('No active workflow');
-        });
-    });
-    (0, vitest_1.describe)('handoff recording', () => {
-        (0, vitest_1.it)('should record handoffs between agents', async () => {
-            await tracer.startWorkflow('Pipeline');
-            await tracer.recordHandoff('Router', 'TechAgent', { issue: 'API' }, 'delegation');
-            const handoffs = tracer.getHandoffs();
-            (0, vitest_1.expect)(handoffs).toHaveLength(1);
-            (0, vitest_1.expect)(handoffs[0].fromAgent).toBe('Router');
-            (0, vitest_1.expect)(handoffs[0].toAgent).toBe('TechAgent');
-            (0, vitest_1.expect)(handoffs[0].handoffType).toBe('delegation');
-            await tracer.endWorkflow();
-        });
-    });
-    (0, vitest_1.describe)('cost tracking', () => {
-        (0, vitest_1.it)('should record and calculate costs', async () => {
-            await tracer.startWorkflow('Pipeline');
-            const cost = await tracer.recordCost({
-                provider: 'openai',
-                model: 'gpt-4o',
-                inputTokens: 1000,
-                outputTokens: 500,
-            });
-            // gpt-4o pricing: $5/1M input, $15/1M output
-            (0, vitest_1.expect)(cost.totalTokens).toBe(1500);
-            (0, vitest_1.expect)(parseFloat(cost.inputCost)).toBeCloseTo(0.005, 4);
-            (0, vitest_1.expect)(parseFloat(cost.outputCost)).toBeCloseTo(0.0075, 4);
-            (0, vitest_1.expect)(parseFloat(cost.totalCost)).toBeCloseTo(0.0125, 4);
-            (0, vitest_1.expect)(tracer.getTotalCost()).toBeCloseTo(0.0125, 4);
-            await tracer.endWorkflow();
-        });
-        (0, vitest_1.it)('should track cost breakdown by category', async () => {
-            await tracer.startWorkflow('Pipeline');
-            await tracer.recordCost({
-                provider: 'openai',
-                model: 'gpt-4o',
-                inputTokens: 1000,
-                outputTokens: 500,
-                category: 'llm',
-            });
-            await tracer.recordCost({
-                provider: 'openai',
-                model: 'gpt-4o',
-                inputTokens: 200,
-                outputTokens: 100,
-                category: 'tool',
-            });
-            const breakdown = tracer.getCostBreakdown();
-            (0, vitest_1.expect)(breakdown.llm).toBeGreaterThan(0);
-            (0, vitest_1.expect)(breakdown.tool).toBeGreaterThan(0);
-            (0, vitest_1.expect)(breakdown.embedding).toBe(0);
-            await tracer.endWorkflow();
-        });
-        (0, vitest_1.it)('should use default pricing for unknown models', async () => {
-            await tracer.startWorkflow('Pipeline');
-            const cost = await tracer.recordCost({
-                provider: 'custom',
-                model: 'unknown-model',
-                inputTokens: 1000000,
-                outputTokens: 1000000,
-            });
-            // Default: $1/1M input, $3/1M output = $4 total
-            (0, vitest_1.expect)(parseFloat(cost.totalCost)).toBeCloseTo(4.0, 1);
-            await tracer.endWorkflow();
-        });
-    });
-    (0, vitest_1.describe)('state resets', () => {
-        (0, vitest_1.it)('should reset state when starting a new workflow', async () => {
-            await tracer.startWorkflow('First');
-            await tracer.recordCost({
-                provider: 'openai',
-                model: 'gpt-4o',
-                inputTokens: 1000,
-                outputTokens: 500,
-            });
-            await tracer.recordHandoff(undefined, 'Agent1');
-            await tracer.recordDecision({
-                agent: 'A',
-                type: 'action',
-                chosen: 'x',
-                alternatives: [],
-            });
-            await tracer.endWorkflow();
-            // Start fresh workflow
-            await tracer.startWorkflow('Second');
-            (0, vitest_1.expect)(tracer.getCosts()).toHaveLength(0);
-            (0, vitest_1.expect)(tracer.getHandoffs()).toHaveLength(0);
-            (0, vitest_1.expect)(tracer.getDecisions()).toHaveLength(0);
-            (0, vitest_1.expect)(tracer.getTotalCost()).toBe(0);
-            await tracer.endWorkflow();
-        });
-    });
-});
-(0, vitest_1.describe)('createWorkflowTracer', () => {
-    (0, vitest_1.beforeEach)(() => {
-        vitest_1.vi.stubGlobal('fetch', vitest_1.vi.fn().mockResolvedValue({
-            ok: true,
-            json: async () => ({ id: 1, traceId: 't', name: 'n' }),
-            status: 200,
-        }));
-    });
-    (0, vitest_1.afterEach)(() => {
-        vitest_1.vi.unstubAllGlobals();
-    });
-    (0, vitest_1.it)('should create a WorkflowTracer instance', () => {
-        const client = new client_1.AIEvalClient({ apiKey: 'key', baseUrl: 'http://localhost:3000' });
-        const tracer = (0, workflows_1.createWorkflowTracer)(client, { organizationId: 1 });
-        (0, vitest_1.expect)(tracer).toBeInstanceOf(workflows_1.WorkflowTracer);
-    });
-});
-(0, vitest_1.describe)('traceWorkflowStep', () => {
-    (0, vitest_1.beforeEach)(() => {
-        vitest_1.vi.stubGlobal('fetch', vitest_1.vi.fn().mockResolvedValue({
-            ok: true,
-            json: async () => ({ id: 1, traceId: 't', name: 'n' }),
-            status: 200,
-        }));
-    });
-    (0, vitest_1.afterEach)(() => {
-        vitest_1.vi.unstubAllGlobals();
-    });
-    (0, vitest_1.it)('should trace a function execution', async () => {
-        const client = new client_1.AIEvalClient({ apiKey: 'key', baseUrl: 'http://localhost:3000', organizationId: 1 });
-        const tracer = new workflows_1.WorkflowTracer(client, { organizationId: 1 });
-        await tracer.startWorkflow('Test');
-        const result = await (0, workflows_1.traceWorkflowStep)(tracer, 'Agent', async () => {
-            return 'output';
-        });
-        (0, vitest_1.expect)(result).toBe('output');
-        await tracer.endWorkflow();
-    });
-    (0, vitest_1.it)('should re-throw errors from the traced function', async () => {
-        const client = new client_1.AIEvalClient({ apiKey: 'key', baseUrl: 'http://localhost:3000', organizationId: 1 });
-        const tracer = new workflows_1.WorkflowTracer(client, { organizationId: 1 });
-        await tracer.startWorkflow('Test');
-        await (0, vitest_1.expect)((0, workflows_1.traceWorkflowStep)(tracer, 'Agent', async () => {
-            throw new Error('agent error');
-        })).rejects.toThrow('agent error');
-        await tracer.endWorkflow({ status: 'failed' }, 'failed');
-    });
-});