@evalgate/sdk 2.0.0

package/dist/integrations/anthropic.js

@@ -0,0 +1,163 @@
+"use strict";
+/**
+ * Anthropic Integration
+ * Tier 1.2: Framework Auto-Instrumentation - Anthropic wrapper
+ *
+ * @example
+ * ```typescript
+ * import { traceAnthropic } from '@ai-eval-platform/sdk/integrations/anthropic';
+ * import Anthropic from '@anthropic-ai/sdk';
+ *
+ * const anthropic = new Anthropic({ apiKey: '...' });
+ * const tracedAnthropic = traceAnthropic(anthropic, client);
+ *
+ * // All calls are automatically traced
+ * const message = await tracedAnthropic.messages.create({
+ *   model: 'claude-3-5-sonnet-20241022',
+ *   max_tokens: 1024,
+ *   messages: [{ role: 'user', content: 'Hello!' }]
+ * });
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.traceAnthropic = traceAnthropic;
+exports.traceAnthropicCall = traceAnthropicCall;
+const context_1 = require("../context");
+/**
+ * Wrap Anthropic client with automatic tracing
+ *
+ * @example
+ * ```typescript
+ * import Anthropic from '@anthropic-ai/sdk';
+ * import { traceAnthropic } from '@ai-eval-platform/sdk/integrations/anthropic';
+ *
+ * const anthropic = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
+ * const tracedAnthropic = traceAnthropic(anthropic, evalClient);
+ *
+ * // Automatically traced
+ * const message = await tracedAnthropic.messages.create({
+ *   model: 'claude-3-5-sonnet-20241022',
+ *   max_tokens: 1024,
+ *   messages: [{ role: 'user', content: 'Hello, Claude!' }]
+ * });
+ * ```
+ */
+function traceAnthropic(anthropic, evalClient, options = {}) {
+    const { captureInput = true, captureOutput = true, captureMetadata = true, organizationId, tracePrefix = "anthropic", } = options;
+    // Create proxy for messages.create
+    const originalCreate = anthropic.messages.create.bind(anthropic.messages);
+    anthropic.messages.create = async (params, requestOptions) => {
+        const startTime = Date.now();
+        const traceId = `${tracePrefix}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+        try {
+            // Call original method
+            const message = await originalCreate(params, requestOptions);
+            const durationMs = Date.now() - startTime;
+            // Create trace with success status and complete metadata
+            const traceMetadata = (0, context_1.mergeWithContext)({
+                model: params.model,
+                temperature: params.temperature,
+                max_tokens: params.max_tokens,
+                ...(captureInput ? { input: params.messages } : {}),
+                ...(captureOutput ? { output: message.content } : {}),
+                ...(captureMetadata
+                    ? {
+                        usage: message.usage,
+                        stop_reason: message.stop_reason,
+                    }
+                    : {}),
+            });
+            await evalClient.traces.create({
+                name: `Anthropic: ${params.model}`,
+                traceId,
+                organizationId: organizationId || evalClient.getOrganizationId(),
+                status: "success",
+                durationMs,
+                metadata: traceMetadata,
+            });
+            return message;
+        }
+        catch (error) {
+            const durationMs = Date.now() - startTime;
+            // Create trace with error status
+            const errorMetadata = (0, context_1.mergeWithContext)({
+                model: params.model,
+                temperature: params.temperature,
+                max_tokens: params.max_tokens,
+                ...(captureInput ? { input: params.messages } : {}),
+                ...(captureMetadata ? { params } : {}),
+                error: error instanceof Error ? error.message : String(error),
+            });
+            await evalClient.traces
+                .create({
+                name: `Anthropic: ${params.model}`,
+                traceId,
+                organizationId: organizationId || evalClient.getOrganizationId(),
+                status: "error",
+                durationMs,
+                metadata: errorMetadata,
+            })
+                .catch(() => {
+                // Ignore errors in trace creation to avoid masking the original error
+            });
+            throw error;
+        }
+    };
+    return anthropic;
+}
+/**
+ * Manual trace wrapper for Anthropic calls
+ *
+ * @example
+ * ```typescript
+ * const message = await traceAnthropicCall(
+ *   evalClient,
+ *   'claude-completion',
+ *   async () => {
+ *     return await anthropic.messages.create({
+ *       model: 'claude-3-5-sonnet-20241022',
+ *       max_tokens: 1024,
+ *       messages: [{ role: 'user', content: 'Hello!' }]
+ *     });
+ *   }
+ * );
+ * ```
+ */
+async function traceAnthropicCall(evalClient, name, fn, options = {}) {
+    const startTime = Date.now();
+    const traceId = `anthropic-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+    try {
+        await evalClient.traces.create({
+            name,
+            traceId,
+            organizationId: options.organizationId || evalClient.getOrganizationId(),
+            status: "pending",
+            metadata: (0, context_1.mergeWithContext)({}),
+        });
+        const result = await fn();
+        const durationMs = Date.now() - startTime;
+        await evalClient.traces.create({
+            name,
+            traceId,
+            organizationId: options.organizationId || evalClient.getOrganizationId(),
+            status: "success",
+            durationMs,
+            metadata: (0, context_1.mergeWithContext)({}),
+        });
+        return result;
+    }
+    catch (error) {
+        const durationMs = Date.now() - startTime;
+        await evalClient.traces.create({
+            name,
+            traceId,
+            organizationId: options.organizationId || evalClient.getOrganizationId(),
+            status: "error",
+            durationMs,
+            metadata: (0, context_1.mergeWithContext)({
+                error: error instanceof Error ? error.message : String(error),
+            }),
+        });
+        throw error;
+    }
+}

package/dist/integrations/openai-eval.d.ts

@@ -0,0 +1,57 @@
+/**
+ * openAIChatEval — One-function OpenAI chat regression testing
+ *
+ * Run local regression tests with OpenAI. No EvalGate account required.
+ * CI-friendly output. Optional reportToEvalGate in v1.5.
+ *
+ * @example
+ * ```typescript
+ * import { openAIChatEval } from '@evalgate/sdk';
+ *
+ * await openAIChatEval({
+ *   name: 'chat-regression',
+ *   cases: [
+ *     { input: 'Hello', expectedOutput: 'greeting' },
+ *     { input: '2 + 2 = ?', expectedOutput: '4' }
+ *   ]
+ * });
+ * ```
+ */
+import type { TestSuiteCaseResult } from "../testing";
+export interface OpenAIChatEvalCase {
+    input: string;
+    expectedOutput?: string;
+    /** Platform test case ID. When provided, used directly for reportToEvalGate (no input matching). */
+    testCaseId?: number;
+    assertions?: ((output: string) => import("../assertions").AssertionResult)[];
+}
+export interface OpenAIChatEvalOptions {
+    name: string;
+    model?: string;
+    apiKey?: string;
+    cases: OpenAIChatEvalCase[];
+    /** Retry failing cases N times (default: 0). Only failing cases are retried. */
+    retries?: number;
+    /** v1.5: Upload results to EvalGate platform for an existing evaluation. Requires evaluationId and EVALGATE_API_KEY. */
+    reportToEvalGate?: boolean;
+    /** Evaluation ID (from config or arg). Required when reportToEvalGate is true. */
+    evaluationId?: string;
+    /** EvalGate API base URL. Default: EVALGATE_BASE_URL or http://localhost:3000 */
+    baseUrl?: string;
+    /** Idempotency key for import (e.g. CI run ID). Prevents duplicate runs on retry. */
+    idempotencyKey?: string;
+}
+export interface OpenAIChatEvalResult {
+    passed: number;
+    total: number;
+    score: number;
+    results: TestSuiteCaseResult[];
+    durationMs: number;
+    /** Case IDs that were retried (flaky recovery) */
+    retriedCases?: string[];
+}
+/**
+ * Run OpenAI chat regression tests locally.
+ * No EvalGate account required. Returns score and prints CI-friendly summary.
+ */
+export declare function openAIChatEval(options: OpenAIChatEvalOptions): Promise<OpenAIChatEvalResult>;

package/dist/integrations/openai-eval.js

@@ -0,0 +1,232 @@
+"use strict";
+/**
+ * openAIChatEval — One-function OpenAI chat regression testing
+ *
+ * Run local regression tests with OpenAI. No EvalGate account required.
+ * CI-friendly output. Optional reportToEvalGate in v1.5.
+ *
+ * @example
+ * ```typescript
+ * import { openAIChatEval } from '@evalgate/sdk';
+ *
+ * await openAIChatEval({
+ *   name: 'chat-regression',
+ *   cases: [
+ *     { input: 'Hello', expectedOutput: 'greeting' },
+ *     { input: '2 + 2 = ?', expectedOutput: '4' }
+ *   ]
+ * });
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.openAIChatEval = openAIChatEval;
+const assertions_1 = require("../assertions");
+const config_1 = require("../cli/config");
+const testing_1 = require("../testing");
+const input_hash_1 = require("../utils/input-hash");
+const MAX_FAILED_CASES_TO_SHOW = 5;
+function getOpenAI() {
+    try {
+        const OpenAI = require("openai");
+        return OpenAI;
+    }
+    catch {
+        throw new Error("openai package is required for openAIChatEval. Install with: npm install openai");
+    }
+}
+function createExecutor(model, apiKey) {
+    const OpenAI = getOpenAI();
+    const openai = new OpenAI({ apiKey });
+    return async (input) => {
+        const response = await openai.chat.completions.create({
+            model,
+            messages: [{ role: "user", content: input }],
+            temperature: 0.1,
+        });
+        return response.choices[0]?.message?.content ?? "";
+    };
+}
+function printSummary(result) {
+    const { passed, total, results } = result;
+    const score = total > 0 ? Math.round((passed / total) * 100) : 0;
+    const failed = results.filter((r) => !r.passed);
+    const status = failed.length === 0 ? "PASS" : "FAIL";
+    console.log(`\n${status} ${passed}/${total}  (score: ${score})\n`);
+    if (failed.length > 0) {
+        const toShow = failed.slice(0, MAX_FAILED_CASES_TO_SHOW);
+        const more = failed.length - toShow.length;
+        console.log(`${failed.length} failing case${failed.length === 1 ? "" : "s"}:`);
+        for (const r of toShow) {
+            const expected = r.expected ?? "(no expected)";
+            console.log(`- "${r.input}" → expected: ${expected}`);
+        }
+        if (more > 0) {
+            console.log(`+ ${more} more`);
+        }
+        console.log("\nGate this in CI:");
+        console.log("  npx -y @evalgate/sdk@^2 init");
+    }
+    else {
+        console.log("Tip: Want dashboards and history?");
+        console.log("Set EVALGATE_API_KEY and connect this to the platform.");
+    }
+}
+/**
+ * Run OpenAI chat regression tests locally.
+ * No EvalGate account required. Returns score and prints CI-friendly summary.
+ */
+async function openAIChatEval(options) {
+    const { name, model = "gpt-4o-mini", apiKey, cases, retries = 0 } = options;
+    const resolvedApiKey = apiKey ?? (typeof process !== "undefined" && process.env?.OPENAI_API_KEY);
+    if (!resolvedApiKey) {
+        throw new Error("OPENAI_API_KEY is required. Set it in the environment or pass apiKey to openAIChatEval.");
+    }
+    const executor = createExecutor(model, resolvedApiKey);
+    const suiteCases = cases.map((c) => {
+        const assertions = c.assertions
+            ? [...c.assertions]
+            : c.expectedOutput
+                ? [
+                    (output) => (0, assertions_1.expect)(output).toContainKeywords(c.expectedOutput?.split(/\s+/).filter(Boolean) || []),
+                ]
+                : undefined;
+        return {
+            input: c.input,
+            expected: c.expectedOutput,
+            assertions,
+        };
+    });
+    const suite = (0, testing_1.createTestSuite)(name, {
+        cases: suiteCases,
+        executor,
+        parallel: true,
+        retries,
+    });
+    const result = await suite.run();
+    const score = result.total > 0 ? Math.round((result.passed / result.total) * 100) : 0;
+    const evalResult = {
+        passed: result.passed,
+        total: result.total,
+        score,
+        results: result.results,
+        durationMs: result.durationMs,
+        ...(result.retriedCases &&
+            result.retriedCases.length > 0 && { retriedCases: result.retriedCases }),
+    };
+    printSummary(evalResult);
+    // v1.5: Optional report to EvalGate platform
+    if (options.reportToEvalGate) {
+        const config = typeof process !== "undefined" && process.cwd
+            ? (0, config_1.loadConfig)(process.cwd())
+            : null;
+        const evalId = options.evaluationId || config?.evaluationId;
+        if (!evalId || String(evalId).trim() === "") {
+            console.log("Run evalgate init and set evaluationId to upload results.");
+            return evalResult;
+        }
+        const evalgateKey = (typeof process !== "undefined" && process.env?.EVALGATE_API_KEY) || "";
+        if (!evalgateKey) {
+            console.log("Set EVALGATE_API_KEY to upload results.");
+            return evalResult;
+        }
+        const baseUrl = options.baseUrl ||
+            config?.baseUrl ||
+            (typeof process !== "undefined" && process.env?.EVALGATE_BASE_URL) ||
+            "http://localhost:3000";
+        const url = String(baseUrl).replace(/\/$/, "");
+        try {
+            // Resolve testCaseId for each result: explicit testCaseId in cases, or match by inputHash
+            const importResults = [];
+            const hasExplicitIds = cases.some((c) => c.testCaseId != null);
+            if (hasExplicitIds) {
+                // Use testCaseId from cases (same order as results)
+                for (let i = 0; i < result.results.length; i++) {
+                    const tcId = cases[i]?.testCaseId;
+                    if (tcId == null) {
+                        console.log("reportToEvalGate: All cases must have testCaseId when unknown has it.");
+                        return evalResult;
+                    }
+                    importResults.push({
+                        testCaseId: tcId,
+                        status: result.results[i].passed ? "passed" : "failed",
+                        output: result.results[i].actual ?? "",
+                        latencyMs: result.results[i].durationMs,
+                    });
+                }
+            }
+            else {
+                // Match by inputHash (same canonicalization as platform)
+                const tcRes = await fetch(`${url}/api/evaluations/${evalId}/test-cases?limit=500`, {
+                    headers: { Authorization: `Bearer ${evalgateKey}` },
+                });
+                if (!tcRes.ok) {
+                    console.log("Could not fetch test cases. Check evaluationId and EVALGATE_API_KEY.");
+                    return evalResult;
+                }
+                const platformCases = (await tcRes.json());
+                const hashToIds = new Map();
+                for (const tc of platformCases) {
+                    const input = tc.input ?? "";
+                    if (!input.trim())
+                        continue;
+                    const hash = (0, input_hash_1.sha256Input)(input);
+                    const existing = hashToIds.get(hash) ?? [];
+                    existing.push(tc.id);
+                    hashToIds.set(hash, existing);
+                }
+                for (const r of result.results) {
+                    const hash = (0, input_hash_1.sha256Input)(r.input ?? "");
+                    const ids = hashToIds.get(hash);
+                    if (ids == null || ids.length === 0) {
+                        console.log(`No platform test case matches input: "${(r.input ?? "").slice(0, 50)}…"`);
+                        return evalResult;
+                    }
+                    if (ids.length > 1) {
+                        console.log(`Multiple platform test cases share the same input (hash collision). Use testCaseId in cases.`);
+                        return evalResult;
+                    }
+                    importResults.push({
+                        testCaseId: ids[0],
+                        status: r.passed ? "passed" : "failed",
+                        output: r.actual ?? "",
+                        latencyMs: r.durationMs,
+                    });
+                }
+            }
+            if (importResults.length !== result.results.length) {
+                console.log("Could not match all results to platform test cases.");
+                return evalResult;
+            }
+            const sdkVersion = "1.4.1";
+            const headers = {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${evalgateKey}`,
+            };
+            if (options.idempotencyKey) {
+                headers["Idempotency-Key"] = options.idempotencyKey;
+            }
+            const importRes = await fetch(`${url}/api/evaluations/${evalId}/runs/import`, {
+                method: "POST",
+                headers,
+                body: JSON.stringify({
+                    environment: "dev",
+                    results: importResults,
+                    importClientVersion: sdkVersion,
+                }),
+            });
+            if (!importRes.ok) {
+                const body = await importRes.text();
+                console.log(`Upload failed: ${importRes.status} — ${body}`);
+                return evalResult;
+            }
+            const importData = (await importRes.json());
+            if (importData.dashboardUrl) {
+                console.log(`Dashboard: ${importData.dashboardUrl}`);
+            }
+        }
+        catch (err) {
+            console.log("Upload failed:", err instanceof Error ? err.message : String(err));
+        }
+    }
+    return evalResult;
+}

package/dist/integrations/openai.d.ts

@@ -0,0 +1,92 @@
+/**
+ * OpenAI Integration
+ * Tier 1.2: Framework Auto-Instrumentation - OpenAI wrapper
+ *
+ * @example
+ * ```typescript
+ * import { traceOpenAI } from '@ai-eval-platform/sdk/integrations/openai';
+ * import OpenAI from 'openai';
+ *
+ * const openai = new OpenAI({ apiKey: '...' });
+ * const tracedOpenAI = traceOpenAI(openai, client);
+ *
+ * // All calls are automatically traced
+ * const response = await tracedOpenAI.chat.completions.create({
+ *   model: 'gpt-4',
+ *   messages: [{ role: 'user', content: 'Hello!' }]
+ * });
+ * ```
+ */
+import type { AIEvalClient } from "../client";
+interface OpenAIChatParams {
+    model: string;
+    messages: unknown[];
+    temperature?: number;
+    max_tokens?: number;
+    [key: string]: unknown;
+}
+interface OpenAIChatCompletion {
+    choices: Array<{
+        message?: unknown;
+        finish_reason?: unknown;
+    }>;
+    usage?: unknown;
+    [key: string]: unknown;
+}
+interface OpenAIClient {
+    chat: {
+        completions: {
+            create: (params: OpenAIChatParams, requestOptions?: Record<string, unknown>) => Promise<OpenAIChatCompletion>;
+        };
+    };
+}
+export interface OpenAITraceOptions {
+    /** Whether to capture input (default: true) */
+    captureInput?: boolean;
+    /** Whether to capture output (default: true) */
+    captureOutput?: boolean;
+    /** Whether to capture metadata (default: true) */
+    captureMetadata?: boolean;
+    /** Organization ID for traces */
+    organizationId?: number;
+    /** Custom trace name prefix */
+    tracePrefix?: string;
+}
+/**
+ * Wrap OpenAI client with automatic tracing
+ *
+ * @example
+ * ```typescript
+ * import OpenAI from 'openai';
+ * import { traceOpenAI } from '@ai-eval-platform/sdk/integrations/openai';
+ *
+ * const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
+ * const tracedOpenAI = traceOpenAI(openai, evalClient);
+ *
+ * // Automatically traced
+ * const completion = await tracedOpenAI.chat.completions.create({
+ *   model: 'gpt-4',
+ *   messages: [{ role: 'user', content: 'Hello!' }]
+ * });
+ * ```
+ */
+export declare function traceOpenAI(openai: OpenAIClient, evalClient: AIEvalClient, options?: OpenAITraceOptions): OpenAIClient;
+/**
+ * Manual trace wrapper for OpenAI calls
+ *
+ * @example
+ * ```typescript
+ * const response = await traceOpenAICall(
+ *   evalClient,
+ *   'gpt-4-completion',
+ *   async () => {
+ *     return await openai.chat.completions.create({
+ *       model: 'gpt-4',
+ *       messages: [{ role: 'user', content: 'Hello!' }]
+ *     });
+ *   }
+ * );
+ * ```
+ */
+export declare function traceOpenAICall<T>(evalClient: AIEvalClient, name: string, fn: () => Promise<T>, options?: OpenAITraceOptions): Promise<T>;
+export {};