@pauly4010/evalai-sdk 1.4.1 → 1.5.5

package/dist/matchers/to-pass-gate.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Vitest/Jest matcher: expect(result).toPassGate()
+ * Use with openAIChatEval: expect(await openAIChatEval(...)).toPassGate()
+ *
+ * @example
+ * ```ts
+ * import { openAIChatEval } from '@pauly4010/evalai-sdk';
+ * import { expect } from 'vitest';
+ * import { extendExpectWithToPassGate } from '@pauly4010/evalai-sdk/matchers';
+ *
+ * extendExpectWithToPassGate(expect);
+ *
+ * it('passes gate', async () => {
+ *   const result = await openAIChatEval({ name: 'test', cases: [...] });
+ *   expect(result).toPassGate();
+ * });
+ * ```
+ */
+import type { OpenAIChatEvalResult } from "../integrations/openai-eval";
+export declare function toPassGate(this: {
+    isNot?: boolean;
+}, received: OpenAIChatEvalResult): {
+    pass: boolean;
+    message: () => string;
+};
+/** Register toPassGate matcher with expect. Call in test setup. */
+export declare function extendExpectWithToPassGate(expect: {
+    extend: (matchers: object) => void;
+}): void;

package/dist/matchers/to-pass-gate.js

@@ -0,0 +1,35 @@
+"use strict";
+/**
+ * Vitest/Jest matcher: expect(result).toPassGate()
+ * Use with openAIChatEval: expect(await openAIChatEval(...)).toPassGate()
+ *
+ * @example
+ * ```ts
+ * import { openAIChatEval } from '@pauly4010/evalai-sdk';
+ * import { expect } from 'vitest';
+ * import { extendExpectWithToPassGate } from '@pauly4010/evalai-sdk/matchers';
+ *
+ * extendExpectWithToPassGate(expect);
+ *
+ * it('passes gate', async () => {
+ *   const result = await openAIChatEval({ name: 'test', cases: [...] });
+ *   expect(result).toPassGate();
+ * });
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toPassGate = toPassGate;
+exports.extendExpectWithToPassGate = extendExpectWithToPassGate;
+function toPassGate(received) {
+    const passed = received.passed === received.total && received.total > 0;
+    const isNot = this.isNot ?? false;
+    const success = isNot ? !passed : passed;
+    const message = () => isNot
+        ? `Expected result not to pass gate (${received.passed}/${received.total} passed, score ${received.score})`
+        : `Expected result to pass gate but ${received.passed}/${received.total} passed (score ${received.score})`;
+    return { pass: success, message };
+}
+/** Register toPassGate matcher with expect. Call in test setup. */
+function extendExpectWithToPassGate(expect) {
+    expect.extend({ toPassGate });
+}

package/dist/pagination.d.ts

@@ -64,7 +64,7 @@ export declare function decodeCursor(cursor: string): any;
 /**
  * Create pagination metadata from response
  */
-export declare function createPaginationMeta<T>(items: T[], limit: number, offset: number, total?: number): PaginatedResponse<T>['pagination'];
+export declare function createPaginationMeta<T>(items: T[], limit: number, offset: number, total?: number): PaginatedResponse<T>["pagination"];
 /**
  * Parse pagination params from cursor or offset
  */

package/dist/pagination.js

@@ -78,11 +78,11 @@ async function* autoPaginate(fetchFn, limit = 50) {
  */
 function encodeCursor(data) {
     const json = JSON.stringify(data);
-    if (typeof globalThis !== 'undefined' && 'btoa' in globalThis) {
+    if (typeof globalThis !== "undefined" && "btoa" in globalThis) {
         return globalThis.btoa(json);
     }
     else {
-        return Buffer.from(json).toString('base64');
+        return Buffer.from(json).toString("base64");
     }
 }
 /**
@@ -91,16 +91,16 @@ function encodeCursor(data) {
 function decodeCursor(cursor) {
     try {
         let json;
-        if (typeof globalThis !== 'undefined' && 'atob' in globalThis) {
+        if (typeof globalThis !== "undefined" && "atob" in globalThis) {
             json = globalThis.atob(cursor);
         }
         else {
-            json = Buffer.from(cursor, 'base64').toString('utf-8');
+            json = Buffer.from(cursor, "base64").toString("utf-8");
         }
         return JSON.parse(json);
     }
-    catch (error) {
-        throw new Error('Invalid cursor format');
+    catch (_error) {
+        throw new Error("Invalid cursor format");
     }
 }
 /**

package/dist/snapshot.js

@@ -58,19 +58,19 @@ exports.compareWithSnapshot = compareWithSnapshot;
 exports.deleteSnapshot = deleteSnapshot;
 exports.listSnapshots = listSnapshots;
 // Environment check
-const isNode = typeof process !== 'undefined' && process.versions?.node;
+const isNode = typeof process !== "undefined" && process.versions?.node;
 if (!isNode) {
-    throw new Error('Snapshot testing requires Node.js and cannot run in browsers. ' +
-        'This feature uses the filesystem for storing snapshots.');
+    throw new Error("Snapshot testing requires Node.js and cannot run in browsers. " +
+        "This feature uses the filesystem for storing snapshots.");
 }
-const fs = __importStar(require("fs"));
-const path = __importStar(require("path"));
-const crypto = __importStar(require("crypto"));
+const crypto = __importStar(require("node:crypto"));
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
 /**
  * Snapshot manager
  */
 class SnapshotManager {
-    constructor(snapshotDir = './.snapshots') {
+    constructor(snapshotDir = "./.snapshots") {
         this.snapshotDir = snapshotDir;
         this.ensureSnapshotDir();
     }
@@ -88,24 +88,24 @@ class SnapshotManager {
     getSnapshotPath(name) {
         // Security: prevent empty names
         if (!name || name.trim().length === 0) {
-            throw new Error('Snapshot name cannot be empty');
+            throw new Error("Snapshot name cannot be empty");
         }
         // Security: prevent path traversal
-        if (name.includes('..') || name.includes('/') || name.includes('\\')) {
+        if (name.includes("..") || name.includes("/") || name.includes("\\")) {
             throw new Error('Snapshot name cannot contain path separators or ".."');
         }
         // Sanitize to alphanumeric, hyphens, and underscores
-        const sanitized = name.replace(/[^a-zA-Z0-9-_]/g, '-');
+        const sanitized = name.replace(/[^a-zA-Z0-9-_]/g, "-");
         // Security: ensure sanitized name is not empty
         if (sanitized.length === 0) {
-            throw new Error('Snapshot name must contain at least one alphanumeric character');
+            throw new Error("Snapshot name must contain at least one alphanumeric character");
         }
         // Security: prevent absolute paths
         const filePath = path.join(this.snapshotDir, `${sanitized}.json`);
         const resolvedPath = path.resolve(filePath);
         const resolvedDir = path.resolve(this.snapshotDir);
         if (!resolvedPath.startsWith(resolvedDir)) {
-            throw new Error('Invalid snapshot path: path traversal detected');
+            throw new Error("Invalid snapshot path: path traversal detected");
         }
         return filePath;
     }
@@ -113,7 +113,7 @@ class SnapshotManager {
      * Generate content hash
      */
     generateHash(content) {
-        return crypto.createHash('sha256').update(content).digest('hex');
+        return crypto.createHash("sha256").update(content).digest("hex");
     }
     /**
      * Save a snapshot
@@ -137,8 +137,8 @@ class SnapshotManager {
                 createdAt: new Date().toISOString(),
                 hash: this.generateHash(output),
                 tags: options?.tags,
-                metadata: options?.metadata
-            }
+                metadata: options?.metadata,
+            },
         };
         fs.writeFileSync(filePath, JSON.stringify(snapshotData, null, 2));
         return snapshotData;
@@ -157,7 +157,7 @@ class SnapshotManager {
         if (!fs.existsSync(filePath)) {
             throw new Error(`Snapshot '${name}' not found`);
         }
-        const content = fs.readFileSync(filePath, 'utf-8');
+        const content = fs.readFileSync(filePath, "utf-8");
         return JSON.parse(content);
     }
     /**
@@ -177,14 +177,14 @@ class SnapshotManager {
         // Exact match check
         const exactMatch = original === currentOutput;
         // Calculate similarity (simple line-based diff)
-        const originalLines = original.split('\n');
-        const currentLines = currentOutput.split('\n');
+        const originalLines = original.split("\n");
+        const currentLines = currentOutput.split("\n");
         const differences = [];
         const maxLines = Math.max(originalLines.length, currentLines.length);
         let matchingLines = 0;
         for (let i = 0; i < maxLines; i++) {
-            const origLine = originalLines[i] || '';
-            const currLine = currentLines[i] || '';
+            const origLine = originalLines[i] || "";
+            const currLine = currentLines[i] || "";
             if (origLine === currLine) {
                 matchingLines++;
             }
@@ -198,7 +198,7 @@ class SnapshotManager {
             similarity,
             differences,
             original,
-            current: currentOutput
+            current: currentOutput,
         };
     }
     /**
@@ -214,8 +214,8 @@ class SnapshotManager {
         const files = fs.readdirSync(this.snapshotDir);
         const snapshots = [];
         for (const file of files) {
-            if (file.endsWith('.json')) {
-                const content = fs.readFileSync(path.join(this.snapshotDir, file), 'utf-8');
+            if (file.endsWith(".json")) {
+                const content = fs.readFileSync(path.join(this.snapshotDir, file), "utf-8");
                 snapshots.push(JSON.parse(content));
             }
         }
@@ -249,7 +249,7 @@ class SnapshotManager {
         return this.save(name, output, {
             tags: existing.metadata.tags,
             metadata: existing.metadata.metadata,
-            overwrite: true
+            overwrite: true,
         });
     }
 }

package/dist/streaming.js

@@ -44,15 +44,15 @@ exports.chunk = chunk;
  * ```
  */
 async function batchProcess(processor, items, options = {}) {
-    const { batchSize = 100, parallel = true, delayMs = 0, onProgress, onError, continueOnError = true } = options;
+    const { batchSize = 100, parallel = true, delayMs = 0, onProgress, onError, continueOnError = true, } = options;
     const result = {
         successful: [],
         failed: [],
         summary: {
             total: items.length,
             successful: 0,
-            failed: 0
-        }
+            failed: 0,
+        },
     };
     // Split into batches
     const batches = [];
@@ -75,11 +75,11 @@ async function batchProcess(processor, items, options = {}) {
                         batch: batchIndex,
                         index: itemIndex,
                         error: error instanceof Error ? error : new Error(String(error)),
-                        item
+                        item,
                     };
                     result.failed.push({
                         item,
-                        error: batchError.error
+                        error: batchError.error,
                     });
                     result.summary.failed++;
                     if (onError)
@@ -107,12 +107,12 @@ async function batchProcess(processor, items, options = {}) {
                 completed: result.summary.successful + result.summary.failed,
                 failed: result.summary.failed,
                 batch: batchIndex + 1,
-                totalBatches: batches.length
+                totalBatches: batches.length,
             });
         }
         // Delay between batches
         if (delayMs > 0 && batchIndex < batches.length - 1) {
-            await new Promise(resolve => setTimeout(resolve, delayMs));
+            await new Promise((resolve) => setTimeout(resolve, delayMs));
         }
     }
     return result;
@@ -146,7 +146,7 @@ async function* streamEvaluation(config) {
                 result,
                 passed: true,
                 completed,
-                total: cases.length
+                total: cases.length,
             };
         }
         catch (error) {
@@ -157,7 +157,7 @@ async function* streamEvaluation(config) {
                 result: error,
                 passed: false,
                 completed,
-                total: cases.length
+                total: cases.length,
             };
         }
     }
@@ -181,7 +181,7 @@ async function batchRead(fetcher, options = {}) {
     while (hasMore && (!maxPages || page < maxPages)) {
         const items = await fetcher({
             limit: pageSize,
-            offset: page * pageSize
+            offset: page * pageSize,
         });
         if (items.length === 0) {
             hasMore = false;
@@ -243,7 +243,7 @@ class RateLimiter {
             const fn = this.queue.shift();
             if (fn) {
                 await fn();
-                await new Promise(resolve => setTimeout(resolve, this.interval));
+                await new Promise((resolve) => setTimeout(resolve, this.interval));
             }
         }
         this.processing = false;

package/dist/testing.d.ts

@@ -21,7 +21,7 @@
  * const results = await suite.run();
  * ```
  */
-import { AssertionResult } from './assertions';
+import { type AssertionResult } from "./assertions";
 /**
  * Test suite case definition (different from API TestCase type)
  * Use this for defining test cases in test suites with assertions
@@ -51,6 +51,8 @@ export interface TestSuiteConfig {
     stopOnFailure?: boolean;
     /** Timeout per test case in ms (default: 30000) */
     timeout?: number;
+    /** Retry failing cases N times (default: 0). Only failing cases are retried. */
+    retries?: number;
 }
 export interface TestSuiteCaseResult {
     /** Test case ID */
@@ -85,6 +87,8 @@ export interface TestSuiteResult {
     durationMs: number;
     /** Individual test results */
     results: TestSuiteCaseResult[];
+    /** Case IDs that were retried (flaky recovery) */
+    retriedCases?: string[];
 }
 /**
  * Test Suite for declarative evaluation testing
@@ -182,7 +186,7 @@ export declare function matchesPattern(pattern: RegExp): (output: string) => Ass
  * });
  * ```
  */
-export declare function hasSentiment(sentiment: 'positive' | 'negative' | 'neutral'): (output: string) => AssertionResult;
+export declare function hasSentiment(sentiment: "positive" | "negative" | "neutral"): (output: string) => AssertionResult;
 /**
  * Helper to create length range assertion
  *

package/dist/testing.js

@@ -59,16 +59,13 @@ class TestSuite {
                 if (this.config.executor) {
                     const timeout = this.config.timeout || 30000;
                     const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error(`Test timeout after ${timeout}ms`)), timeout));
-                    actual = await Promise.race([
-                        this.config.executor(testCase.input),
-                        timeoutPromise
-                    ]);
+                    actual = await Promise.race([this.config.executor(testCase.input), timeoutPromise]);
                 }
                 else if (testCase.expected) {
                     actual = testCase.expected; // Use expected as actual if no executor
                 }
                 else {
-                    throw new Error('No executor provided and no expected output');
+                    throw new Error("No executor provided and no expected output");
                 }
                 // Run assertions
                 const assertions = [];
@@ -97,7 +94,7 @@ class TestSuite {
                     actual,
                     passed: allPassed,
                     assertions,
-                    durationMs
+                    durationMs,
                 };
             }
             catch (error) {
@@ -106,17 +103,17 @@ class TestSuite {
                     id,
                     input: testCase.input,
                     expected: testCase.expected,
-                    actual: '',
+                    actual: "",
                     passed: false,
                     assertions: [],
                     durationMs,
-                    error: error instanceof Error ? error.message : String(error)
+                    error: error instanceof Error ? error.message : String(error),
                 };
             }
         };
         // Run tests
         if (this.config.parallel) {
-            results.push(...await Promise.all(this.config.cases.map((tc, i) => runTestCase(tc, i))));
+            results.push(...(await Promise.all(this.config.cases.map((tc, i) => runTestCase(tc, i)))));
         }
         else {
             for (let i = 0; i < this.config.cases.length; i++) {
@@ -127,16 +124,37 @@ class TestSuite {
                 }
             }
         }
+        const retriedCases = [];
+        const retries = this.config.retries ?? 0;
+        if (retries > 0 && results.length > 0) {
+            const failingIndices = results.map((r, i) => (r.passed ? -1 : i)).filter((i) => i >= 0);
+            for (let attempt = 0; attempt < retries && failingIndices.length > 0; attempt++) {
+                const toRetry = [...failingIndices];
+                failingIndices.length = 0;
+                for (const i of toRetry) {
+                    const tc = this.config.cases[i];
+                    const retryResult = await runTestCase(tc, i);
+                    if (retryResult.passed) {
+                        results[i] = retryResult;
+                        retriedCases.push(retryResult.id);
+                    }
+                    else {
+                        failingIndices.push(i);
+                    }
+                }
+            }
+        }
         const durationMs = Date.now() - startTime;
-        const passed = results.filter(r => r.passed).length;
-        const failed = results.filter(r => !r.passed).length;
+        const passed = results.filter((r) => r.passed).length;
+        const failed = results.filter((r) => !r.passed).length;
         return {
             name: this.name,
             total: results.length,
             passed,
             failed,
             durationMs,
-            results
+            results,
+            ...(retriedCases.length > 0 && { retriedCases }),
         };
     }
     /**

package/dist/types.d.ts

@@ -14,13 +14,13 @@ export interface ClientConfig {
     /** Debug mode - enables request/response logging (default: false) */
     debug?: boolean;
     /** Log level for debug mode (default: 'info') */
-    logLevel?: 'trace' | 'debug' | 'info' | 'warn' | 'error';
+    logLevel?: "trace" | "debug" | "info" | "warn" | "error";
     /** Retry configuration */
     retry?: {
         /** Maximum retry attempts (default: 3) */
         maxAttempts?: number;
         /** Backoff strategy (default: 'exponential') */
-        backoff?: 'exponential' | 'linear' | 'fixed';
+        backoff?: "exponential" | "linear" | "fixed";
         /** Retryable error codes */
         retryableErrors?: string[];
     };
@@ -55,7 +55,7 @@ export declare const EvaluationTemplates: {
     readonly CODE_GENERATION: "code-generation";
     readonly SUMMARIZATION: "summarization";
 };
-export type EvaluationTemplateType = typeof EvaluationTemplates[keyof typeof EvaluationTemplates];
+export type EvaluationTemplateType = (typeof EvaluationTemplates)[keyof typeof EvaluationTemplates];
 /**
  * Feature usage limits for per-organization quotas
  */
@@ -95,7 +95,7 @@ export interface Trace<TMetadata = Record<string, any>> {
     name: string;
     traceId: string;
     organizationId: number;
-    status: 'pending' | 'success' | 'error';
+    status: "pending" | "success" | "error";
     durationMs: number | null;
     metadata: TMetadata | null;
     createdAt: string;
@@ -107,7 +107,7 @@ export interface CreateTraceParams<TMetadata = Record<string, any>> {
     name: string;
     traceId: string;
     organizationId?: number;
-    status?: 'pending' | 'success' | 'error';
+    status?: "pending" | "success" | "error";
     durationMs?: number;
     metadata?: TMetadata;
 }
@@ -115,7 +115,7 @@ export interface CreateTraceParams<TMetadata = Record<string, any>> {
  * Parameters for updating an existing trace
  */
 export interface UpdateTraceParams<TMetadata = Record<string, any>> {
-    status?: 'pending' | 'success' | 'error';
+    status?: "pending" | "success" | "error";
     durationMs?: number;
     metadata?: TMetadata;
 }
@@ -126,7 +126,7 @@ export interface ListTracesParams {
     limit?: number;
     offset?: number;
     organizationId?: number;
-    status?: 'pending' | 'success' | 'error';
+    status?: "pending" | "success" | "error";
     search?: string;
 }
 /**
@@ -164,7 +164,7 @@ export interface Evaluation<TMetadata = Record<string, any>> {
     name: string;
     description: string | null;
     type: string;
-    status: 'draft' | 'active' | 'archived';
+    status: "draft" | "active" | "archived";
     organizationId: number;
     createdBy: number;
     createdAt: string;
@@ -180,7 +180,7 @@ export interface CreateEvaluationParams {
     type: string;
     organizationId?: number;
     createdBy: number;
-    status?: 'draft' | 'active' | 'archived';
+    status?: "draft" | "active" | "archived";
 }
 /**
  * Parameters for updating an evaluation
@@ -189,7 +189,7 @@ export interface UpdateEvaluationParams {
     name?: string;
     description?: string;
     type?: string;
-    status?: 'draft' | 'active' | 'archived';
+    status?: "draft" | "active" | "archived";
 }
 /**
  * Parameters for listing evaluations
@@ -199,7 +199,7 @@ export interface ListEvaluationsParams {
     offset?: number;
     organizationId?: number;
     type?: string;
-    status?: 'draft' | 'active' | 'archived';
+    status?: "draft" | "active" | "archived";
     search?: string;
 }
 /**
@@ -227,7 +227,7 @@ export interface CreateTestCaseParams {
 export interface EvaluationRun {
     id: number;
     evaluationId: number;
-    status: 'pending' | 'running' | 'completed' | 'failed';
+    status: "pending" | "running" | "completed" | "failed";
     results: Record<string, any> | null;
     createdAt: string;
     completedAt: string | null;
@@ -236,7 +236,7 @@ export interface EvaluationRun {
  * Parameters for creating an evaluation run
  */
 export interface CreateRunParams {
-    status?: 'pending' | 'running' | 'completed' | 'failed';
+    status?: "pending" | "running" | "completed" | "failed";
     results?: Record<string, any>;
 }
 /**
@@ -300,7 +300,7 @@ export type LLMJudgeData = LLMJudgeResult;
 export type AnnotationData = any;
 export interface RetryConfig {
     maxAttempts?: number;
-    backoff?: 'exponential' | 'linear' | 'fixed';
+    backoff?: "exponential" | "linear" | "fixed";
     retryableErrors?: string[];
 }
 export interface GenericMetadata {
@@ -327,7 +327,7 @@ export interface SnapshotData {
     updatedAt: string;
 }
 export interface ExportOptions {
-    format: 'json' | 'csv' | 'jsonl';
+    format: "json" | "csv" | "jsonl";
     includeTraces?: boolean;
     includeEvaluations?: boolean;
     includeTestCases?: boolean;
@@ -360,7 +360,7 @@ export interface BatchOptions {
     }) => void;
     signal?: AbortSignal;
 }
-export type ExportFormat = 'json' | 'csv' | 'jsonl';
+export type ExportFormat = "json" | "csv" | "jsonl";
 /**
  * Annotation object representing human feedback
  */
@@ -412,7 +412,7 @@ export interface AnnotationTask {
     description: string | null;
     instructions: string | null;
     type: string;
-    status: 'pending' | 'in_progress' | 'completed' | 'archived';
+    status: "pending" | "in_progress" | "completed" | "archived";
     organizationId: number;
     annotationSettings: Record<string, any>;
     createdAt: string;
@@ -434,7 +434,7 @@ export interface CreateAnnotationTaskParams {
  */
 export interface ListAnnotationTasksParams {
     organizationId?: number;
-    status?: 'pending' | 'in_progress' | 'completed' | 'archived';
+    status?: "pending" | "in_progress" | "completed" | "archived";
     limit?: number;
     offset?: number;
 }
@@ -536,7 +536,7 @@ export interface Webhook {
     url: string;
     events: string[];
     secret: string;
-    status: 'active' | 'inactive';
+    status: "active" | "inactive";
     lastTriggeredAt: string | null;
     createdAt: string;
     updatedAt: string;
@@ -555,14 +555,14 @@ export interface CreateWebhookParams {
 export interface UpdateWebhookParams {
     url?: string;
     events?: string[];
-    status?: 'active' | 'inactive';
+    status?: "active" | "inactive";
 }
 /**
  * Parameters for listing webhooks
  */
 export interface ListWebhooksParams {
     organizationId: number;
-    status?: 'active' | 'inactive';
+    status?: "active" | "inactive";
     limit?: number;
     offset?: number;
 }
@@ -716,7 +716,7 @@ export interface Organization {
     name: string;
     slug: string;
     plan: string;
-    status: 'active' | 'suspended' | 'cancelled';
+    status: "active" | "suspended" | "cancelled";
     createdAt: string;
     updatedAt: string;
     metadata?: Record<string, any>;