@evalgate/sdk 2.0.0

package/dist/batch.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Request batching for improved performance
+ * Combines multiple API requests into fewer network calls
+ */
+export interface BatchRequest {
+    id: string;
+    method: string;
+    endpoint: string;
+    body?: unknown;
+    headers?: Record<string, string>;
+}
+export interface BatchResponse {
+    id: string;
+    status: number;
+    data?: unknown;
+    error?: string;
+}
+/**
+ * Batch processor for API requests
+ */
+export declare class RequestBatcher {
+    private executeBatch;
+    private queue;
+    private batchTimer;
+    private readonly maxBatchSize;
+    private readonly batchDelay;
+    private requestCounter;
+    constructor(executeBatch: (requests: BatchRequest[]) => Promise<BatchResponse[]>, options?: {
+        maxBatchSize?: number;
+        batchDelay?: number;
+    });
+    /**
+     * Add request to batch queue
+     */
+    enqueue(method: string, endpoint: string, body?: unknown, headers?: Record<string, string>): Promise<unknown>;
+    /**
+     * Schedule batch processing after delay
+     */
+    private scheduleBatch;
+    /**
+     * Process current batch
+     */
+    private processBatch;
+    /**
+     * Flush all pending requests immediately
+     */
+    flush(): Promise<void>;
+    /**
+     * Clear queue without processing
+     */
+    clear(): void;
+    /**
+     * Get queue statistics
+     */
+    getStats(): {
+        queueSize: number;
+        maxBatchSize: number;
+    };
+}
+/**
+ * @internal - Internal SDK logic, not part of public API
+ * Check if requests can be batched together
+ */
+export declare function canBatch(method: string, endpoint: string): boolean;
+/**
+ * Batch multiple async operations with concurrency limit
+ */
+export declare function batchProcess<T, R>(items: T[], processor: (item: T) => Promise<R>, concurrency?: number): Promise<R[]>;

package/dist/batch.js

@@ -0,0 +1,179 @@
+"use strict";
+/**
+ * Request batching for improved performance
+ * Combines multiple API requests into fewer network calls
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RequestBatcher = void 0;
+exports.canBatch = canBatch;
+exports.batchProcess = batchProcess;
+/**
+ * Batch processor for API requests
+ */
+class RequestBatcher {
+    constructor(executeBatch, options = {}) {
+        this.executeBatch = executeBatch;
+        this.queue = [];
+        this.batchTimer = null;
+        this.requestCounter = 0;
+        this.maxBatchSize = options.maxBatchSize || 10;
+        this.batchDelay = options.batchDelay || 50; // 50ms
+    }
+    /**
+     * Add request to batch queue
+     */
+    async enqueue(method, endpoint, body, headers) {
+        return new Promise((resolve, reject) => {
+            const id = `req_${this.requestCounter++}_${Date.now()}`;
+            this.queue.push({
+                id,
+                resolve,
+                reject,
+                request: { id, method, endpoint, body, headers },
+            });
+            // Process immediately if batch is full
+            if (this.queue.length >= this.maxBatchSize) {
+                this.processBatch();
+            }
+            else {
+                // Otherwise schedule batch processing
+                this.scheduleBatch();
+            }
+        });
+    }
+    /**
+     * Schedule batch processing after delay
+     */
+    scheduleBatch() {
+        if (this.batchTimer) {
+            return;
+        }
+        this.batchTimer = setTimeout(() => {
+            this.processBatch();
+        }, this.batchDelay);
+    }
+    /**
+     * Process current batch
+     */
+    async processBatch() {
+        if (this.batchTimer) {
+            if (typeof this.batchTimer === "number") {
+                clearTimeout(this.batchTimer);
+            }
+            else {
+                clearTimeout(this.batchTimer);
+            }
+            this.batchTimer = null;
+        }
+        if (this.queue.length === 0) {
+            return;
+        }
+        // Take items from queue
+        const batch = this.queue.splice(0, this.maxBatchSize);
+        const requests = batch.map((item) => item.request);
+        try {
+            const responses = await this.executeBatch(requests);
+            // Match responses to requests and resolve/reject
+            for (const response of responses) {
+                const pendingRequest = batch.find((item) => item.id === response.id);
+                if (pendingRequest) {
+                    if (response.status >= 200 && response.status < 300) {
+                        pendingRequest.resolve(response.data);
+                    }
+                    else {
+                        pendingRequest.reject(new Error(response.error ||
+                            `Request failed with status ${response.status}`));
+                    }
+                }
+            }
+            // Handle unknown requests that didn't get a response
+            for (const item of batch) {
+                if (!responses.find((r) => r.id === item.id)) {
+                    item.reject(new Error("No response received for request"));
+                }
+            }
+        }
+        catch (error) {
+            // Reject all requests in batch on error
+            for (const item of batch) {
+                item.reject(error);
+            }
+        }
+        // If there are more items in queue, schedule next batch
+        if (this.queue.length > 0) {
+            this.scheduleBatch();
+        }
+    }
+    /**
+     * Flush all pending requests immediately
+     */
+    async flush() {
+        while (this.queue.length > 0) {
+            await this.processBatch();
+        }
+    }
+    /**
+     * Clear queue without processing
+     */
+    clear() {
+        if (this.batchTimer) {
+            if (typeof this.batchTimer === "number") {
+                clearTimeout(this.batchTimer);
+            }
+            else {
+                clearTimeout(this.batchTimer);
+            }
+            this.batchTimer = null;
+        }
+        // Reject all pending requests
+        for (const item of this.queue) {
+            item.reject(new Error("Batch queue cleared"));
+        }
+        this.queue = [];
+    }
+    /**
+     * Get queue statistics
+     */
+    getStats() {
+        return {
+            queueSize: this.queue.length,
+            maxBatchSize: this.maxBatchSize,
+        };
+    }
+}
+exports.RequestBatcher = RequestBatcher;
+/**
+ * @internal - Internal SDK logic, not part of public API
+ * Check if requests can be batched together
+ */
+function canBatch(method, endpoint) {
+    if (method !== "GET") {
+        return false;
+    }
+    const batchableEndpoints = [
+        "/traces",
+        "/evaluations",
+        "/annotations",
+        "/results",
+    ];
+    return batchableEndpoints.some((pattern) => endpoint.includes(pattern));
+}
+/**
+ * Batch multiple async operations with concurrency limit
+ */
+async function batchProcess(items, processor, concurrency = 5) {
+    const results = [];
+    const executing = [];
+    for (const item of items) {
+        const promise = processor(item).then((result) => {
+            results.push(result);
+        });
+        executing.push(promise);
+        if (executing.length >= concurrency) {
+            await Promise.race(executing);
+            executing.splice(executing.indexOf(promise), 1);
+        }
+    }
+    await Promise.all(executing);
+    return results;
+}

package/dist/cache.d.ts

@@ -0,0 +1,65 @@
+/**
+ * Simple in-memory cache with TTL for SDK requests
+ * Reduces redundant API calls and improves performance
+ */
+export declare class RequestCache {
+    private cache;
+    private maxSize;
+    constructor(maxSize?: number);
+    /**
+     * Generate cache key from request parameters
+     */
+    private generateKey;
+    /**
+     * Check if cache entry is still valid
+     */
+    private isValid;
+    /**
+     * Get cached response if valid
+     */
+    get<T>(method: string, url: string, params?: unknown): T | null;
+    /**
+     * Store response in cache
+     */
+    set<T>(method: string, url: string, data: T, ttl: number, params?: unknown): void;
+    /**
+     * Invalidate specific cache entry
+     */
+    invalidate(method: string, url: string, params?: unknown): void;
+    /**
+     * Invalidate all cache entries matching a pattern
+     */
+    invalidatePattern(pattern: string): void;
+    /**
+     * Clear all cache entries
+     */
+    clear(): void;
+    /**
+     * Get cache statistics
+     */
+    getStats(): {
+        size: number;
+        maxSize: number;
+        hitRate?: number;
+    };
+}
+/**
+ * Default cache TTL values (in milliseconds)
+ * @internal - Used by SDK internally, exposed for advanced customization only
+ */
+export declare const CacheTTL: {
+    readonly SHORT: number;
+    readonly MEDIUM: number;
+    readonly LONG: number;
+    readonly HOUR: number;
+};
+/**
+ * @internal - Internal SDK logic, not part of public API
+ * Determine if a request should be cached based on method and endpoint
+ */
+export declare function shouldCache(method: string, endpoint: string): boolean;
+/**
+ * @internal - Internal SDK logic, not part of public API
+ * Get appropriate TTL for an endpoint
+ */
+export declare function getTTL(endpoint: string): number;

package/dist/cache.js

@@ -0,0 +1,131 @@
+"use strict";
+/**
+ * Simple in-memory cache with TTL for SDK requests
+ * Reduces redundant API calls and improves performance
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CacheTTL = exports.RequestCache = void 0;
+exports.shouldCache = shouldCache;
+exports.getTTL = getTTL;
+class RequestCache {
+    constructor(maxSize = 1000) {
+        this.cache = new Map();
+        this.maxSize = maxSize;
+    }
+    /**
+     * Generate cache key from request parameters
+     */
+    generateKey(method, url, params) {
+        const paramString = params ? JSON.stringify(params) : "";
+        return `${method}:${url}:${paramString}`;
+    }
+    /**
+     * Check if cache entry is still valid
+     */
+    isValid(entry) {
+        return Date.now() - entry.timestamp < entry.ttl;
+    }
+    /**
+     * Get cached response if valid
+     */
+    get(method, url, params) {
+        const key = this.generateKey(method, url, params);
+        const entry = this.cache.get(key);
+        if (!entry) {
+            return null;
+        }
+        if (!this.isValid(entry)) {
+            this.cache.delete(key);
+            return null;
+        }
+        return entry.data;
+    }
+    /**
+     * Store response in cache
+     */
+    set(method, url, data, ttl, params) {
+        // Enforce cache size limit (LRU-style)
+        if (this.cache.size >= this.maxSize) {
+            const firstKey = this.cache.keys().next().value;
+            if (firstKey) {
+                this.cache.delete(firstKey);
+            }
+        }
+        const key = this.generateKey(method, url, params);
+        this.cache.set(key, {
+            data,
+            timestamp: Date.now(),
+            ttl,
+        });
+    }
+    /**
+     * Invalidate specific cache entry
+     */
+    invalidate(method, url, params) {
+        const key = this.generateKey(method, url, params);
+        this.cache.delete(key);
+    }
+    /**
+     * Invalidate all cache entries matching a pattern
+     */
+    invalidatePattern(pattern) {
+        for (const key of Array.from(this.cache.keys())) {
+            if (key.includes(pattern)) {
+                this.cache.delete(key);
+            }
+        }
+    }
+    /**
+     * Clear all cache entries
+     */
+    clear() {
+        this.cache.clear();
+    }
+    /**
+     * Get cache statistics
+     */
+    getStats() {
+        return {
+            size: this.cache.size,
+            maxSize: this.maxSize,
+        };
+    }
+}
+exports.RequestCache = RequestCache;
+/**
+ * Default cache TTL values (in milliseconds)
+ * @internal - Used by SDK internally, exposed for advanced customization only
+ */
+exports.CacheTTL = {
+    SHORT: 30 * 1000,
+    MEDIUM: 5 * 60 * 1000,
+    LONG: 30 * 60 * 1000,
+    HOUR: 60 * 60 * 1000,
+};
+/**
+ * @internal - Internal SDK logic, not part of public API
+ * Determine if a request should be cached based on method and endpoint
+ */
+function shouldCache(method, endpoint) {
+    if (method !== "GET") {
+        return false;
+    }
+    const noCacheEndpoints = ["/health", "/usage", "/deliveries"];
+    return !noCacheEndpoints.some((pattern) => endpoint.includes(pattern));
+}
+/**
+ * @internal - Internal SDK logic, not part of public API
+ * Get appropriate TTL for an endpoint
+ */
+function getTTL(endpoint) {
+    if (endpoint.includes("/api-keys") || endpoint.includes("/webhooks")) {
+        return exports.CacheTTL.LONG;
+    }
+    if (endpoint.includes("/evaluations") || endpoint.includes("/configs")) {
+        return exports.CacheTTL.MEDIUM;
+    }
+    if (endpoint.includes("/traces") || endpoint.includes("/results")) {
+        return exports.CacheTTL.SHORT;
+    }
+    return exports.CacheTTL.MEDIUM;
+}

package/dist/cli/api.d.ts

@@ -0,0 +1,108 @@
+/**
+ * API fetch helpers for evalgate check.
+ * Captures x-request-id from response headers.
+ * Sends X-EvalGate-SDK-Version and X-EvalGate-Spec-Version on all requests.
+ */
+export type QualityLatestData = {
+    score?: number;
+    total?: number | null;
+    evidenceLevel?: string | null;
+    baselineScore?: number | null;
+    regressionDelta?: number | null;
+    baselineMissing?: boolean | null;
+    breakdown?: {
+        passRate?: number;
+        safety?: number;
+        judge?: number;
+    };
+    flags?: string[];
+    evaluationRunId?: number;
+    evaluationId?: number;
+    avgLatencyMs?: number | null;
+    costUsd?: number | null;
+    baselineCostUsd?: number | null;
+    baselineRunId?: number | null;
+};
+export type RunDetailsData = {
+    results?: Array<{
+        testCaseId?: number;
+        status?: string;
+        output?: string;
+        durationMs?: number;
+        assertionsJson?: Record<string, unknown>;
+        test_cases?: {
+            name?: string;
+            input?: string;
+            expectedOutput?: string;
+        };
+    }>;
+};
+export declare function fetchQualityLatest(baseUrl: string, apiKey: string, evaluationId: string, baseline: string): Promise<{
+    ok: true;
+    data: QualityLatestData;
+    requestId?: string;
+} | {
+    ok: false;
+    status: number;
+    body: string;
+    requestId?: string;
+}>;
+export declare function fetchRunDetails(baseUrl: string, apiKey: string, evaluationId: string, runId: number): Promise<{
+    ok: true;
+    data: RunDetailsData;
+} | {
+    ok: false;
+}>;
+export type CiContext = {
+    provider?: "github" | "gitlab" | "circle" | "unknown";
+    repo?: string;
+    sha?: string;
+    branch?: string;
+    pr?: number;
+    runUrl?: string;
+    actor?: string;
+};
+export type ImportResult = {
+    testCaseId: number;
+    status: "passed" | "failed";
+    output: string;
+    latencyMs?: number;
+    costUsd?: number;
+    assertionsJson?: Record<string, unknown>;
+};
+export type PublishShareResult = {
+    shareId: string;
+    shareUrl: string;
+    shareScope: string;
+};
+export declare function fetchRunExport(baseUrl: string, apiKey: string, evaluationId: string, runId: number): Promise<{
+    ok: true;
+    exportData: Record<string, unknown>;
+} | {
+    ok: false;
+    status: number;
+    body: string;
+}>;
+export declare function publishShare(baseUrl: string, apiKey: string, evaluationId: string, exportData: Record<string, unknown>, evaluationRunId: number, options?: {
+    expiresInDays?: number;
+}): Promise<{
+    ok: true;
+    data: PublishShareResult;
+} | {
+    ok: false;
+    status: number;
+    body: string;
+}>;
+export declare function importRunOnFail(baseUrl: string, apiKey: string, evaluationId: string, results: ImportResult[], options: {
+    idempotencyKey?: string;
+    ci?: CiContext;
+    importClientVersion?: string;
+    checkReport?: Record<string, unknown>;
+}): Promise<{
+    ok: true;
+    runId: number;
+} | {
+    ok: false;
+    status: number;
+    body: string;
+}>;

package/dist/cli/api.js

@@ -0,0 +1,132 @@
+"use strict";
+/**
+ * API fetch helpers for evalgate check.
+ * Captures x-request-id from response headers.
+ * Sends X-EvalGate-SDK-Version and X-EvalGate-Spec-Version on all requests.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchQualityLatest = fetchQualityLatest;
+exports.fetchRunDetails = fetchRunDetails;
+exports.fetchRunExport = fetchRunExport;
+exports.publishShare = publishShare;
+exports.importRunOnFail = importRunOnFail;
+const version_1 = require("../version");
+const API_HEADERS = {
+    "X-EvalGate-SDK-Version": version_1.SDK_VERSION,
+    "X-EvalGate-Spec-Version": version_1.SPEC_VERSION,
+};
+async function fetchQualityLatest(baseUrl, apiKey, evaluationId, baseline) {
+    const headers = { ...API_HEADERS, Authorization: `Bearer ${apiKey}` };
+    const url = `${baseUrl.replace(/\/$/, "")}/api/quality?evaluationId=${evaluationId}&action=latest&baseline=${baseline}`;
+    try {
+        const res = await fetch(url, { headers });
+        const requestId = res.headers.get("x-request-id") ?? undefined;
+        const body = await res.text();
+        if (!res.ok) {
+            return { ok: false, status: res.status, body, requestId };
+        }
+        const data = JSON.parse(body);
+        return { ok: true, data, requestId };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return { ok: false, status: 0, body: msg, requestId: undefined };
+    }
+}
+async function fetchRunDetails(baseUrl, apiKey, evaluationId, runId) {
+    const headers = { ...API_HEADERS, Authorization: `Bearer ${apiKey}` };
+    const url = `${baseUrl.replace(/\/$/, "")}/api/evaluations/${evaluationId}/runs/${runId}`;
+    try {
+        const res = await fetch(url, { headers });
+        if (!res.ok)
+            return { ok: false };
+        const data = (await res.json());
+        return { ok: true, data };
+    }
+    catch {
+        return { ok: false };
+    }
+}
+async function fetchRunExport(baseUrl, apiKey, evaluationId, runId) {
+    const headers = { ...API_HEADERS, Authorization: `Bearer ${apiKey}` };
+    const url = `${baseUrl.replace(/\/$/, "")}/api/evaluations/${evaluationId}/runs/${runId}/export`;
+    try {
+        const res = await fetch(url, { headers });
+        const text = await res.text();
+        if (!res.ok)
+            return { ok: false, status: res.status, body: text };
+        const exportData = JSON.parse(text);
+        return { ok: true, exportData };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return { ok: false, status: 0, body: msg };
+    }
+}
+async function publishShare(baseUrl, apiKey, evaluationId, exportData, evaluationRunId, options) {
+    const headers = {
+        ...API_HEADERS,
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json",
+    };
+    const body = {
+        exportData,
+        shareScope: "run",
+        evaluationRunId,
+        ...(options?.expiresInDays != null && {
+            expiresInDays: options.expiresInDays,
+        }),
+    };
+    const url = `${baseUrl.replace(/\/$/, "")}/api/evaluations/${evaluationId}/publish`;
+    try {
+        const res = await fetch(url, {
+            method: "POST",
+            headers,
+            body: JSON.stringify(body),
+        });
+        const text = await res.text();
+        if (!res.ok)
+            return { ok: false, status: res.status, body: text };
+        const data = JSON.parse(text);
+        return { ok: true, data };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return { ok: false, status: 0, body: msg };
+    }
+}
+async function importRunOnFail(baseUrl, apiKey, evaluationId, results, options) {
+    const headers = {
+        ...API_HEADERS,
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json",
+    };
+    if (options.idempotencyKey) {
+        headers["Idempotency-Key"] = options.idempotencyKey;
+    }
+    const body = {
+        environment: "dev",
+        results,
+        importClientVersion: options.importClientVersion ?? "evalgate-cli",
+        ci: options.ci,
+        ...(options.checkReport != null && { checkReport: options.checkReport }),
+    };
+    const url = `${baseUrl.replace(/\/$/, "")}/api/evaluations/${evaluationId}/runs/import`;
+    try {
+        const res = await fetch(url, {
+            method: "POST",
+            headers,
+            body: JSON.stringify(body),
+        });
+        const text = await res.text();
+        if (!res.ok) {
+            return { ok: false, status: res.status, body: text };
+        }
+        const data = JSON.parse(text);
+        return { ok: true, runId: data.runId };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return { ok: false, status: 0, body: msg };
+    }
+}

package/dist/cli/baseline.d.ts

@@ -0,0 +1,10 @@
+/**
+ * evalai baseline — Baseline management commands
+ *
+ * Subcommands:
+ *   evalgate baseline init    — Create a starter evals/baseline.json
+ *   evalgate baseline update  — Run tests + update baseline with real scores
+ */
+export declare function runBaselineInit(cwd: string): number;
+export declare function runBaselineUpdate(cwd: string): number;
+export declare function runBaseline(argv: string[]): number;