@pauly4010/evalai-sdk 1.9.0 → 1.9.1

package/dist/client.request.test.js

@@ -1,157 +1,232 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
+var __createBinding =
+	(this && this.__createBinding) ||
+	(Object.create
+		? function (o, m, k, k2) {
+				if (k2 === undefined) k2 = k;
+				var desc = Object.getOwnPropertyDescriptor(m, k);
+				if (
+					!desc ||
+					("get" in desc ? !m.__esModule : desc.writable || desc.configurable)
+				) {
+					desc = {
+						enumerable: true,
+						get: function () {
+							return m[k];
+						},
+					};
+				}
+				Object.defineProperty(o, k2, desc);
+			}
+		: function (o, m, k, k2) {
+				if (k2 === undefined) k2 = k;
+				o[k2] = m[k];
+			});
+var __setModuleDefault =
+	(this && this.__setModuleDefault) ||
+	(Object.create
+		? function (o, v) {
+				Object.defineProperty(o, "default", { enumerable: true, value: v });
+			}
+		: function (o, v) {
+				o["default"] = v;
+			});
+var __importStar =
+	(this && this.__importStar) ||
+	(function () {
+		var ownKeys = function (o) {
+			ownKeys =
+				Object.getOwnPropertyNames ||
+				function (o) {
+					var ar = [];
+					for (var k in o)
+						if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+					return ar;
+				};
+			return ownKeys(o);
+		};
+		return function (mod) {
+			if (mod && mod.__esModule) return mod;
+			var result = {};
+			if (mod != null)
+				for (var k = ownKeys(mod), i = 0; i < k.length; i++)
+					if (k[i] !== "default") __createBinding(result, mod, k[i]);
+			__setModuleDefault(result, mod);
+			return result;
+		};
+	})();
 Object.defineProperty(exports, "__esModule", { value: true });
 const vitest_1 = require("vitest");
 const client_1 = require("./client");
 const errorsModule = __importStar(require("./errors"));
 vitest_1.vi.mock("./cache", () => {
-    const cacheTracker = { invalidatedPatterns: [] };
-    const shouldCache = vitest_1.vi.fn().mockReturnValue(true);
-    const getTTL = vitest_1.vi.fn().mockReturnValue(1000);
-    const makeKey = (method, url, params) => `${method}:${url}:${JSON.stringify(params ?? null)}`;
-    return {
-        __esModule: true,
-        shouldCache,
-        getTTL,
-        cacheTracker,
-        RequestCache: class RequestCache {
-            constructor() {
-                this.store = new Map();
-            }
-            get(method, url, params) {
-                const key = makeKey(method, url, params);
-                return this.store.get(key) ?? null;
-            }
-            set(method, url, data, _ttl, params) {
-                const key = makeKey(method, url, params);
-                this.store.set(key, data);
-            }
-            invalidatePattern(pattern) {
-                cacheTracker.invalidatedPatterns.push(pattern);
-            }
-            invalidate(_method, _url, _params) {
-                // no-op for tests
-            }
-            clear() {
-                this.store.clear();
-            }
-        },
-    };
+	const cacheTracker = { invalidatedPatterns: [] };
+	const shouldCache = vitest_1.vi.fn().mockReturnValue(true);
+	const getTTL = vitest_1.vi.fn().mockReturnValue(1000);
+	const makeKey = (method, url, params) =>
+		`${method}:${url}:${JSON.stringify(params ?? null)}`;
+	return {
+		__esModule: true,
+		shouldCache,
+		getTTL,
+		cacheTracker,
+		RequestCache: class RequestCache {
+			constructor() {
+				this.store = new Map();
+			}
+			get(method, url, params) {
+				const key = makeKey(method, url, params);
+				return this.store.get(key) ?? null;
+			}
+			set(method, url, data, _ttl, params) {
+				const key = makeKey(method, url, params);
+				this.store.set(key, data);
+			}
+			invalidatePattern(pattern) {
+				cacheTracker.invalidatedPatterns.push(pattern);
+			}
+			invalidate(_method, _url, _params) {
+				// no-op for tests
+			}
+			clear() {
+				this.store.clear();
+			}
+		},
+	};
 });
 const cache_1 = require("./cache");
 (0, vitest_1.describe)("AIEvalClient.request", () => {
-    (0, vitest_1.beforeEach)(() => {
-        process.env.EVALAI_API_KEY = "test";
-        cache_1.shouldCache.mockReset().mockReturnValue(true);
-        cache_1.getTTL.mockReset().mockReturnValue(1000);
-        cache_1.cacheTracker.invalidatedPatterns.length = 0;
-    });
-    (0, vitest_1.it)("caches GET responses and reuses data without re-fetching", async () => {
-        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost", timeout: 1000 });
-        const payload = { items: [1, 2, 3] };
-        const fetchMock = vitest_1.vi.fn().mockResolvedValue({
-            ok: true,
-            status: 200,
-            json: async () => payload,
-        });
-        globalThis.fetch = fetchMock;
-        const first = await client.request("/api/traces", { method: "GET" });
-        const second = await client.request("/api/traces", { method: "GET" });
-        (0, vitest_1.expect)(first).toEqual(payload);
-        (0, vitest_1.expect)(second).toEqual(payload);
-        (0, vitest_1.expect)(fetchMock).toHaveBeenCalledTimes(1);
-    });
-    (0, vitest_1.it)("propagates non-ok responses as SDK errors", async () => {
-        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost" });
-        const fetchMock = vitest_1.vi.fn().mockResolvedValue({
-            ok: false,
-            status: 429,
-            json: async () => ({ error: { code: "RATE_LIMIT_EXCEEDED" } }),
-        });
-        globalThis.fetch = fetchMock;
-        const createErrorSpy = vitest_1.vi
-            .spyOn(errorsModule, "createErrorFromResponse")
-            .mockReturnValue(new errorsModule.EvalAIError("rate limited", "RATE_LIMIT_EXCEEDED", 429));
-        await (0, vitest_1.expect)(client.request("/api/fail", { method: "GET" })).rejects.toHaveProperty("code", "RATE_LIMIT_EXCEEDED");
-        createErrorSpy.mockRestore();
-    });
-    (0, vitest_1.it)("retries on retryable SDK errors and eventually succeeds", async () => {
-        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost", timeout: 1000 });
-        vitest_1.vi.spyOn(client, "calculateBackoff").mockReturnValue(0);
-        const failureResponse = {
-            ok: false,
-            status: 429,
-            json: async () => ({ error: { code: "RATE_LIMIT_EXCEEDED" } }),
-        };
-        const successResponse = {
-            ok: true,
-            status: 200,
-            json: async () => ({ ok: true }),
-        };
-        const createErrorSpy = vitest_1.vi
-            .spyOn(errorsModule, "createErrorFromResponse")
-            .mockReturnValue(new errorsModule.EvalAIError("rate limited", "RATE_LIMIT_EXCEEDED", 429));
-        const fetchMock = vitest_1.vi
-            .fn()
-            .mockResolvedValueOnce(failureResponse)
-            .mockResolvedValueOnce(successResponse);
-        globalThis.fetch = fetchMock;
-        const result = await client.request("/api/retry", { method: "GET" });
-        (0, vitest_1.expect)(result).toEqual({ ok: true });
-        (0, vitest_1.expect)(fetchMock).toHaveBeenCalledTimes(2);
-        createErrorSpy.mockRestore();
-    });
-    (0, vitest_1.it)("throws a TIMEOUT SDK error when fetch aborts", async () => {
-        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost", timeout: 1000 });
-        const abortError = Object.assign(new Error("aborted"), { name: "AbortError" });
-        const fetchMock = vitest_1.vi.fn().mockRejectedValue(abortError);
-        globalThis.fetch = fetchMock;
-        await (0, vitest_1.expect)(client.request("/api/timeout", { method: "GET" })).rejects.toMatchObject({
-            code: "TIMEOUT",
-        });
-    });
-    (0, vitest_1.it)("invalidates related cache entries for mutation requests", async () => {
-        const client = new client_1.AIEvalClient({ apiKey: "test", baseUrl: "http://localhost", timeout: 1000 });
-        cache_1.shouldCache.mockReturnValue(false);
-        const fetchMock = vitest_1.vi.fn().mockResolvedValue({
-            ok: true,
-            status: 201,
-            json: async () => ({ result: "ok" }),
-        });
-        globalThis.fetch = fetchMock;
-        await client.request("/api/evaluations", { method: "POST", body: JSON.stringify({}) });
-        (0, vitest_1.expect)(cache_1.cacheTracker.invalidatedPatterns).toContain("evaluations");
-    });
+	(0, vitest_1.beforeEach)(() => {
+		process.env.EVALAI_API_KEY = "test";
+		cache_1.shouldCache.mockReset().mockReturnValue(true);
+		cache_1.getTTL.mockReset().mockReturnValue(1000);
+		cache_1.cacheTracker.invalidatedPatterns.length = 0;
+	});
+	(0, vitest_1.it)(
+		"caches GET responses and reuses data without re-fetching",
+		async () => {
+			const client = new client_1.AIEvalClient({
+				apiKey: "test",
+				baseUrl: "http://localhost",
+				timeout: 1000,
+			});
+			const payload = { items: [1, 2, 3] };
+			const fetchMock = vitest_1.vi.fn().mockResolvedValue({
+				ok: true,
+				status: 200,
+				json: async () => payload,
+			});
+			globalThis.fetch = fetchMock;
+			const first = await client.request("/api/traces", { method: "GET" });
+			const second = await client.request("/api/traces", { method: "GET" });
+			(0, vitest_1.expect)(first).toEqual(payload);
+			(0, vitest_1.expect)(second).toEqual(payload);
+			(0, vitest_1.expect)(fetchMock).toHaveBeenCalledTimes(1);
+		},
+	);
+	(0, vitest_1.it)("propagates non-ok responses as SDK errors", async () => {
+		const client = new client_1.AIEvalClient({
+			apiKey: "test",
+			baseUrl: "http://localhost",
+		});
+		const fetchMock = vitest_1.vi.fn().mockResolvedValue({
+			ok: false,
+			status: 429,
+			json: async () => ({ error: { code: "RATE_LIMIT_EXCEEDED" } }),
+		});
+		globalThis.fetch = fetchMock;
+		const createErrorSpy = vitest_1.vi
+			.spyOn(errorsModule, "createErrorFromResponse")
+			.mockReturnValue(
+				new errorsModule.EvalAIError(
+					"rate limited",
+					"RATE_LIMIT_EXCEEDED",
+					429,
+				),
+			);
+		await (0, vitest_1.expect)(
+			client.request("/api/fail", { method: "GET" }),
+		).rejects.toHaveProperty("code", "RATE_LIMIT_EXCEEDED");
+		createErrorSpy.mockRestore();
+	});
+	(0, vitest_1.it)(
+		"retries on retryable SDK errors and eventually succeeds",
+		async () => {
+			const client = new client_1.AIEvalClient({
+				apiKey: "test",
+				baseUrl: "http://localhost",
+				timeout: 1000,
+			});
+			vitest_1.vi.spyOn(client, "calculateBackoff").mockReturnValue(0);
+			const failureResponse = {
+				ok: false,
+				status: 429,
+				json: async () => ({ error: { code: "RATE_LIMIT_EXCEEDED" } }),
+			};
+			const successResponse = {
+				ok: true,
+				status: 200,
+				json: async () => ({ ok: true }),
+			};
+			const createErrorSpy = vitest_1.vi
+				.spyOn(errorsModule, "createErrorFromResponse")
+				.mockReturnValue(
+					new errorsModule.EvalAIError(
+						"rate limited",
+						"RATE_LIMIT_EXCEEDED",
+						429,
+					),
+				);
+			const fetchMock = vitest_1.vi
+				.fn()
+				.mockResolvedValueOnce(failureResponse)
+				.mockResolvedValueOnce(successResponse);
+			globalThis.fetch = fetchMock;
+			const result = await client.request("/api/retry", { method: "GET" });
+			(0, vitest_1.expect)(result).toEqual({ ok: true });
+			(0, vitest_1.expect)(fetchMock).toHaveBeenCalledTimes(2);
+			createErrorSpy.mockRestore();
+		},
+	);
+	(0, vitest_1.it)("throws a TIMEOUT SDK error when fetch aborts", async () => {
+		const client = new client_1.AIEvalClient({
+			apiKey: "test",
+			baseUrl: "http://localhost",
+			timeout: 1000,
+		});
+		const abortError = Object.assign(new Error("aborted"), {
+			name: "AbortError",
+		});
+		const fetchMock = vitest_1.vi.fn().mockRejectedValue(abortError);
+		globalThis.fetch = fetchMock;
+		await (0, vitest_1.expect)(
+			client.request("/api/timeout", { method: "GET" }),
+		).rejects.toMatchObject({
+			code: "TIMEOUT",
+		});
+	});
+	(0, vitest_1.it)(
+		"invalidates related cache entries for mutation requests",
+		async () => {
+			const client = new client_1.AIEvalClient({
+				apiKey: "test",
+				baseUrl: "http://localhost",
+				timeout: 1000,
+			});
+			cache_1.shouldCache.mockReturnValue(false);
+			const fetchMock = vitest_1.vi.fn().mockResolvedValue({
+				ok: true,
+				status: 201,
+				json: async () => ({ result: "ok" }),
+			});
+			globalThis.fetch = fetchMock;
+			await client.request("/api/evaluations", {
+				method: "POST",
+				body: JSON.stringify({}),
+			});
+			(0, vitest_1.expect)(cache_1.cacheTracker.invalidatedPatterns).toContain(
+				"evaluations",
+			);
+		},
+	);
 });

package/dist/context.js

@@ -28,7 +28,9 @@ exports.withContext = withContext;
 exports.withContextSync = withContextSync;
 exports.WithContext = WithContext;
 // Detect environment
-const isNode = typeof process !== "undefined" && process.versions?.node && typeof require !== "undefined";
+const isNode = typeof process !== "undefined" &&
+    process.versions?.node &&
+    typeof require !== "undefined";
 // Browser fallback: simple context stack
 class BrowserContextStorage {
     constructor() {

package/dist/errors.js

@@ -162,8 +162,11 @@ class EvalAIError extends Error {
         this.details = details;
         // Initialize required properties from ERROR_DOCS
         const doc = ERROR_DOCS[code];
-        this.documentation = doc?.documentation ?? `https://docs.ai-eval-platform.com/errors/${code}`;
-        this.solutions = doc?.solutions ?? ["Check the error details for more information"];
+        this.documentation =
+            doc?.documentation ?? `https://docs.ai-eval-platform.com/errors/${code}`;
+        this.solutions = doc?.solutions ?? [
+            "Check the error details for more information",
+        ];
         this.retryable = doc?.retryable ?? false;
         // Extract retry-after for rate limits
         const errorDetails = details;
@@ -174,7 +177,8 @@ class EvalAIError extends Error {
         if (code === "FEATURE_LIMIT_REACHED" && errorDetails?.resetAt) {
             this.resetAt = new Date(errorDetails.resetAt);
         }
-        this.requestId = errorDetails?.error?.requestId ?? errorDetails?.requestId;
+        this.requestId =
+            errorDetails?.error?.requestId ?? errorDetails?.requestId;
         // Ensure proper prototype chain
         Object.setPrototypeOf(this, EvalAIError.prototype);
     }
@@ -236,7 +240,10 @@ function createErrorFromResponse(response, data) {
     const message = typeof errorData?.error === "string"
         ? errorData.error
         : (errObj?.message ?? errorData?.message ?? response.statusText);
-    const requestId = errObj?.requestId ?? errorData?.requestId ?? response.headers.get("x-request-id") ?? undefined;
+    const requestId = errObj?.requestId ??
+        errorData?.requestId ??
+        response.headers.get("x-request-id") ??
+        undefined;
     // Map HTTP status to error codes when code not in response
     if (!errObj?.code && !errorData?.code) {
         if (status === 401)

package/dist/export.js

@@ -300,7 +300,9 @@ async function importFromLangSmith(client, langsmithData, options) {
             traceId: run.id || `langsmith-${Date.now()}-${Math.random()}`,
             organizationId: options.organizationId,
             status: run.error ? "error" : "success",
-            durationMs: run.execution_time ? Math.round(run.execution_time * 1000) : null,
+            durationMs: run.execution_time
+                ? Math.round(run.execution_time * 1000)
+                : null,
             metadata: {
                 source: "langsmith",
                 original_id: run.id,

package/dist/index.d.ts

@@ -13,13 +13,13 @@ NetworkError, };
 export { containsAllRequiredFields, containsJSON, containsKeywords, containsLanguage, expect, followsInstructions, hasFactualAccuracy, hasLength, hasNoHallucinations, hasNoToxicity, hasReadabilityScore, hasSentiment, hasValidCodeSyntax, isValidEmail, isValidURL, matchesPattern, matchesSchema, notContainsPII, respondedWithinTime, similarTo, withinRange, } from "./assertions";
 import { createContext, EvalContext, getCurrentContext, withContext } from "./context";
 export { createContext, getCurrentContext as getContext, withContext, EvalContext as ContextManager, };
-export { createTestSuite, type TestCaseResult, TestSuite, TestSuiteCase, TestSuiteCaseResult, TestSuiteConfig, TestSuiteResult, } from "./testing";
-export { defineEval, evalai, defineSuite, createContext as createEvalContext, createResult, } from "./runtime/eval";
-export { createEvalRuntime, getActiveRuntime, setActiveRuntime, disposeActiveRuntime, } from "./runtime/registry";
+export { cloneContext, mergeContexts, validateContext, } from "./runtime/context";
+export { createContext as createEvalContext, createResult, defineEval, defineSuite, evalai, } from "./runtime/eval";
 export { createLocalExecutor, defaultLocalExecutor, } from "./runtime/executor";
-export { mergeContexts, cloneContext, validateContext, } from "./runtime/context";
-export type { EvalSpec, EvalContext, EvalResult, EvalOptions, EvalRuntime, EvalExecutor, EvalExecutorInterface, LocalExecutor, CloudExecutor, WorkerExecutor, SpecConfig, SpecOptions, DefineEvalFunction, ExecutorCapabilities, } from "./runtime/types";
-export { EvalRuntimeError, SpecRegistrationError, SpecExecutionError, RuntimeError, } from "./runtime/types";
+export { createEvalRuntime, disposeActiveRuntime, getActiveRuntime, setActiveRuntime, } from "./runtime/registry";
+export type { CloudExecutor, DefineEvalFunction, EvalContext, EvalExecutor, EvalExecutorInterface, EvalOptions, EvalResult, EvalRuntime, EvalSpec, ExecutorCapabilities, LocalExecutor, SpecConfig, SpecOptions, WorkerExecutor, } from "./runtime/types";
+export { EvalRuntimeError, RuntimeError, SpecExecutionError, SpecRegistrationError, } from "./runtime/types";
+export { createTestSuite, type TestCaseResult, TestSuite, TestSuiteCase, TestSuiteCaseResult, TestSuiteConfig, TestSuiteResult, } from "./testing";
 import { compareWithSnapshot, snapshot } from "./snapshot";
 export { snapshot, compareWithSnapshot, snapshot as saveSnapshot, compareWithSnapshot as compareSnapshots, };
 import type { ExportFormat } from "./export";
@@ -36,8 +36,8 @@ export { Logger } from "./logger";
 export { extendExpectWithToPassGate } from "./matchers";
 export { autoPaginate, createPaginatedIterator, decodeCursor, encodeCursor, PaginatedIterator, type PaginatedResponse, type PaginationParams, } from "./pagination";
 export { ARTIFACTS, type Baseline, type BaselineTolerance, GATE_CATEGORY, GATE_EXIT, type GateCategory, type GateExitCode, REPORT_SCHEMA_VERSION, type RegressionDelta, type RegressionReport, } from "./regression";
-export { batchProcess, batchRead, RateLimiter, streamEvaluation } from "./streaming";
-export type { Annotation, AnnotationItem, AnnotationTask, APIKey, APIKeyUsage, APIKeyWithSecret, BatchOptions, ClientConfig as AIEvalConfig, CreateAnnotationItemParams, CreateAnnotationParams, CreateAnnotationTaskParams, CreateAPIKeyParams, CreateLLMJudgeConfigParams, CreateWebhookParams, Evaluation as EvaluationData, ExportOptions, GenericMetadata as AnnotationData, GetLLMJudgeAlignmentParams, GetUsageParams, ImportOptions, ListAnnotationItemsParams, ListAnnotationsParams, ListAnnotationTasksParams, ListAPIKeysParams, ListLLMJudgeConfigsParams, ListLLMJudgeResultsParams, ListWebhookDeliveriesParams, ListWebhooksParams, LLMJudgeAlignment, LLMJudgeConfig, LLMJudgeResult as LLMJudgeData, Organization, RetryConfig, SnapshotData, Span as SpanData, StreamOptions, TestCase, TestResult, Trace as TraceData, TracedResponse, UpdateAPIKeyParams, UpdateWebhookParams, UsageStats, UsageSummary, Webhook, WebhookDelivery, } from "./types";
+export { batchProcess, batchRead, RateLimiter, streamEvaluation, } from "./streaming";
+export type { Annotation, AnnotationItem, AnnotationTask, APIKey, APIKeyUsage, APIKeyWithSecret, BatchOptions, ClientConfig as AIEvalConfig, CreateAnnotationItemParams, CreateAnnotationParams, CreateAnnotationTaskParams, CreateAPIKeyParams, CreateLLMJudgeConfigParams, CreateWebhookParams, Evaluation as EvaluationData, EvaluationRun, EvaluationRunDetail, ExportOptions, GenericMetadata as AnnotationData, GetLLMJudgeAlignmentParams, GetUsageParams, ImportOptions, ListAnnotationItemsParams, ListAnnotationsParams, ListAnnotationTasksParams, ListAPIKeysParams, ListLLMJudgeConfigsParams, ListLLMJudgeResultsParams, ListWebhookDeliveriesParams, ListWebhooksParams, LLMJudgeAlignment, LLMJudgeConfig, LLMJudgeEvaluateResult, LLMJudgeResult as LLMJudgeData, Organization, RetryConfig, SnapshotData, Span as SpanData, StreamOptions, TestCase, TestResult, Trace as TraceData, TraceDetail, TracedResponse, UpdateAPIKeyParams, UpdateWebhookParams, UsageStats, UsageSummary, Webhook, WebhookDelivery, } from "./types";
 export { EvaluationTemplates, type EvaluationTemplateType, type FeatureUsage, type OrganizationLimits, } from "./types";
 export { type AgentHandoff, type AgentSpanContext, type CostCategory, type CostRecord, createWorkflowTracer, type DecisionAlternative, type DecisionType, type HandoffType, type LLMProvider, type RecordCostParams, type RecordDecisionParams, traceAutoGen, traceCrewAI, traceLangChainAgent, traceWorkflowStep, type WorkflowContext, type WorkflowDefinition, type WorkflowEdge, type WorkflowNode, type WorkflowStatus, WorkflowTracer, type WorkflowTracerOptions, } from "./workflows";
 import { AIEvalClient } from "./client";

package/dist/index.js

@@ -8,8 +8,8 @@
  * @packageDocumentation
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SpecExecutionError = exports.SpecRegistrationError = exports.EvalRuntimeError = exports.validateContext = exports.cloneContext = exports.mergeContexts = exports.defaultLocalExecutor = exports.createLocalExecutor = exports.disposeActiveRuntime = exports.setActiveRuntime = exports.getActiveRuntime = exports.createEvalRuntime = exports.createResult = exports.createEvalContext = exports.defineSuite = exports.evalai = exports.defineEval = exports.TestSuite = exports.createTestSuite = exports.ContextManager = exports.withContext = exports.getContext = exports.createContext = exports.withinRange = exports.similarTo = exports.respondedWithinTime = exports.notContainsPII = exports.matchesSchema = exports.matchesPattern = exports.isValidURL = exports.isValidEmail = exports.hasValidCodeSyntax = exports.hasSentiment = exports.hasReadabilityScore = exports.hasNoToxicity = exports.hasNoHallucinations = exports.hasLength = exports.hasFactualAccuracy = exports.followsInstructions = exports.expect = exports.containsLanguage = exports.containsKeywords = exports.containsJSON = exports.containsAllRequiredFields = exports.NetworkError = exports.ValidationError = exports.AuthenticationError = exports.RateLimitError = exports.EvalAIError = exports.AIEvalClient = void 0;
-exports.WorkflowTracer = exports.traceWorkflowStep = exports.traceLangChainAgent = exports.traceCrewAI = exports.traceAutoGen = exports.createWorkflowTracer = exports.EvaluationTemplates = exports.streamEvaluation = exports.RateLimiter = exports.batchRead = exports.batchProcess = exports.REPORT_SCHEMA_VERSION = exports.GATE_EXIT = exports.GATE_CATEGORY = exports.ARTIFACTS = exports.PaginatedIterator = exports.encodeCursor = exports.decodeCursor = exports.createPaginatedIterator = exports.autoPaginate = exports.extendExpectWithToPassGate = exports.Logger = exports.openAIChatEval = exports.traceOpenAI = exports.traceAnthropic = exports.runCheck = exports.parseArgs = exports.EXIT = exports.RequestCache = exports.CacheTTL = exports.RequestBatcher = exports.importData = exports.exportData = exports.compareSnapshots = exports.saveSnapshot = exports.compareWithSnapshot = exports.snapshot = exports.RuntimeError = void 0;
+exports.createTestSuite = exports.SpecRegistrationError = exports.SpecExecutionError = exports.RuntimeError = exports.EvalRuntimeError = exports.setActiveRuntime = exports.getActiveRuntime = exports.disposeActiveRuntime = exports.createEvalRuntime = exports.defaultLocalExecutor = exports.createLocalExecutor = exports.evalai = exports.defineSuite = exports.defineEval = exports.createResult = exports.createEvalContext = exports.validateContext = exports.mergeContexts = exports.cloneContext = exports.ContextManager = exports.withContext = exports.getContext = exports.createContext = exports.withinRange = exports.similarTo = exports.respondedWithinTime = exports.notContainsPII = exports.matchesSchema = exports.matchesPattern = exports.isValidURL = exports.isValidEmail = exports.hasValidCodeSyntax = exports.hasSentiment = exports.hasReadabilityScore = exports.hasNoToxicity = exports.hasNoHallucinations = exports.hasLength = exports.hasFactualAccuracy = exports.followsInstructions = exports.expect = exports.containsLanguage = exports.containsKeywords = exports.containsJSON = exports.containsAllRequiredFields = exports.NetworkError = exports.ValidationError = exports.AuthenticationError = exports.RateLimitError = exports.EvalAIError = exports.AIEvalClient = void 0;
+exports.WorkflowTracer = exports.traceWorkflowStep = exports.traceLangChainAgent = exports.traceCrewAI = exports.traceAutoGen = exports.createWorkflowTracer = exports.EvaluationTemplates = exports.streamEvaluation = exports.RateLimiter = exports.batchRead = exports.batchProcess = exports.REPORT_SCHEMA_VERSION = exports.GATE_EXIT = exports.GATE_CATEGORY = exports.ARTIFACTS = exports.PaginatedIterator = exports.encodeCursor = exports.decodeCursor = exports.createPaginatedIterator = exports.autoPaginate = exports.extendExpectWithToPassGate = exports.Logger = exports.openAIChatEval = exports.traceOpenAI = exports.traceAnthropic = exports.runCheck = exports.parseArgs = exports.EXIT = exports.RequestCache = exports.CacheTTL = exports.RequestBatcher = exports.importData = exports.exportData = exports.compareSnapshots = exports.saveSnapshot = exports.compareWithSnapshot = exports.snapshot = exports.TestSuite = void 0;
 // Main SDK exports
 var client_1 = require("./client");
 Object.defineProperty(exports, "AIEvalClient", { enumerable: true, get: function () { return client_1.AIEvalClient; } });
@@ -49,35 +49,35 @@ Object.defineProperty(exports, "createContext", { enumerable: true, get: functio
 Object.defineProperty(exports, "ContextManager", { enumerable: true, get: function () { return context_1.EvalContext; } });
 Object.defineProperty(exports, "getContext", { enumerable: true, get: function () { return context_1.getCurrentContext; } });
 Object.defineProperty(exports, "withContext", { enumerable: true, get: function () { return context_1.withContext; } });
-// Test suite builder (Tier 2.7) - BACKWARD COMPATIBILITY LAYER
-var testing_1 = require("./testing");
-Object.defineProperty(exports, "createTestSuite", { enumerable: true, get: function () { return testing_1.createTestSuite; } });
-Object.defineProperty(exports, "TestSuite", { enumerable: true, get: function () { return testing_1.TestSuite; } });
+var context_2 = require("./runtime/context");
+Object.defineProperty(exports, "cloneContext", { enumerable: true, get: function () { return context_2.cloneContext; } });
+Object.defineProperty(exports, "mergeContexts", { enumerable: true, get: function () { return context_2.mergeContexts; } });
+Object.defineProperty(exports, "validateContext", { enumerable: true, get: function () { return context_2.validateContext; } });
 // LAYER 1: Runtime Foundation - NEW PROGRAMMING MODEL
 var eval_1 = require("./runtime/eval");
-Object.defineProperty(exports, "defineEval", { enumerable: true, get: function () { return eval_1.defineEval; } });
-Object.defineProperty(exports, "evalai", { enumerable: true, get: function () { return eval_1.evalai; } });
-Object.defineProperty(exports, "defineSuite", { enumerable: true, get: function () { return eval_1.defineSuite; } });
 Object.defineProperty(exports, "createEvalContext", { enumerable: true, get: function () { return eval_1.createContext; } });
 Object.defineProperty(exports, "createResult", { enumerable: true, get: function () { return eval_1.createResult; } });
+Object.defineProperty(exports, "defineEval", { enumerable: true, get: function () { return eval_1.defineEval; } });
+Object.defineProperty(exports, "defineSuite", { enumerable: true, get: function () { return eval_1.defineSuite; } });
+Object.defineProperty(exports, "evalai", { enumerable: true, get: function () { return eval_1.evalai; } });
+var executor_1 = require("./runtime/executor");
+Object.defineProperty(exports, "createLocalExecutor", { enumerable: true, get: function () { return executor_1.createLocalExecutor; } });
+Object.defineProperty(exports, "defaultLocalExecutor", { enumerable: true, get: function () { return executor_1.defaultLocalExecutor; } });
 var registry_1 = require("./runtime/registry");
 Object.defineProperty(exports, "createEvalRuntime", { enumerable: true, get: function () { return registry_1.createEvalRuntime; } });
+Object.defineProperty(exports, "disposeActiveRuntime", { enumerable: true, get: function () { return registry_1.disposeActiveRuntime; } });
 Object.defineProperty(exports, "getActiveRuntime", { enumerable: true, get: function () { return registry_1.getActiveRuntime; } });
 Object.defineProperty(exports, "setActiveRuntime", { enumerable: true, get: function () { return registry_1.setActiveRuntime; } });
-Object.defineProperty(exports, "disposeActiveRuntime", { enumerable: true, get: function () { return registry_1.disposeActiveRuntime; } });
-var executor_1 = require("./runtime/executor");
-Object.defineProperty(exports, "createLocalExecutor", { enumerable: true, get: function () { return executor_1.createLocalExecutor; } });
-Object.defineProperty(exports, "defaultLocalExecutor", { enumerable: true, get: function () { return executor_1.defaultLocalExecutor; } });
-var context_2 = require("./runtime/context");
-Object.defineProperty(exports, "mergeContexts", { enumerable: true, get: function () { return context_2.mergeContexts; } });
-Object.defineProperty(exports, "cloneContext", { enumerable: true, get: function () { return context_2.cloneContext; } });
-Object.defineProperty(exports, "validateContext", { enumerable: true, get: function () { return context_2.validateContext; } });
 // Runtime errors
 var types_1 = require("./runtime/types");
 Object.defineProperty(exports, "EvalRuntimeError", { enumerable: true, get: function () { return types_1.EvalRuntimeError; } });
-Object.defineProperty(exports, "SpecRegistrationError", { enumerable: true, get: function () { return types_1.SpecRegistrationError; } });
-Object.defineProperty(exports, "SpecExecutionError", { enumerable: true, get: function () { return types_1.SpecExecutionError; } });
 Object.defineProperty(exports, "RuntimeError", { enumerable: true, get: function () { return types_1.RuntimeError; } });
+Object.defineProperty(exports, "SpecExecutionError", { enumerable: true, get: function () { return types_1.SpecExecutionError; } });
+Object.defineProperty(exports, "SpecRegistrationError", { enumerable: true, get: function () { return types_1.SpecRegistrationError; } });
+// Test suite builder (Tier 2.7) - BACKWARD COMPATIBILITY LAYER
+var testing_1 = require("./testing");
+Object.defineProperty(exports, "createTestSuite", { enumerable: true, get: function () { return testing_1.createTestSuite; } });
+Object.defineProperty(exports, "TestSuite", { enumerable: true, get: function () { return testing_1.TestSuite; } });
 // Snapshot testing (Tier 2.8)
 const snapshot_1 = require("./snapshot");
 Object.defineProperty(exports, "compareWithSnapshot", { enumerable: true, get: function () { return snapshot_1.compareWithSnapshot; } });

package/dist/integrations/anthropic.d.ts

@@ -19,6 +19,24 @@
  * ```
  */
 import type { AIEvalClient } from "../client";
+interface AnthropicMessageParams {
+    model: string;
+    messages: unknown[];
+    temperature?: number;
+    max_tokens?: number;
+    [key: string]: unknown;
+}
+interface AnthropicMessage {
+    content: unknown;
+    usage?: unknown;
+    stop_reason?: unknown;
+    [key: string]: unknown;
+}
+interface AnthropicClient {
+    messages: {
+        create: (params: AnthropicMessageParams, requestOptions?: Record<string, unknown>) => Promise<AnthropicMessage>;
+    };
+}
 export interface AnthropicTraceOptions {
     /** Whether to capture input (default: true) */
     captureInput?: boolean;
@@ -50,7 +68,7 @@ export interface AnthropicTraceOptions {
  * });
  * ```
  */
-export declare function traceAnthropic(anthropic: any, evalClient: AIEvalClient, options?: AnthropicTraceOptions): any;
+export declare function traceAnthropic(anthropic: AnthropicClient, evalClient: AIEvalClient, options?: AnthropicTraceOptions): AnthropicClient;
 /**
  * Manual trace wrapper for Anthropic calls
  *
@@ -70,3 +88,4 @@ export declare function traceAnthropic(anthropic: any, evalClient: AIEvalClient,
  * ```
  */
 export declare function traceAnthropicCall<T>(evalClient: AIEvalClient, name: string, fn: () => Promise<T>, options?: AnthropicTraceOptions): Promise<T>;
+export {};

package/dist/integrations/openai-eval.js

@@ -87,7 +87,7 @@ async function openAIChatEval(options) {
             ? [...c.assertions]
             : c.expectedOutput
                 ? [
-                    (output) => (0, assertions_1.expect)(output).toContainKeywords(c.expectedOutput.split(/\s+/).filter(Boolean)),
+                    (output) => (0, assertions_1.expect)(output).toContainKeywords(c.expectedOutput?.split(/\s+/).filter(Boolean) || []),
                 ]
                 : undefined;
         return {
@@ -116,7 +116,9 @@ async function openAIChatEval(options) {
     printSummary(evalResult);
     // v1.5: Optional report to EvalAI platform
     if (options.reportToEvalAI) {
-        const config = typeof process !== "undefined" && process.cwd ? (0, config_1.loadConfig)(process.cwd()) : null;
+        const config = typeof process !== "undefined" && process.cwd
+            ? (0, config_1.loadConfig)(process.cwd())
+            : null;
         const evalId = options.evaluationId || config?.evaluationId;
         if (!evalId || String(evalId).trim() === "") {
             console.log("Run evalai init and set evaluationId to upload results.");