@evalgate/sdk 2.0.0

package/dist/integrations/openai.js

@@ -0,0 +1,160 @@
+"use strict";
+/**
+ * OpenAI Integration
+ * Tier 1.2: Framework Auto-Instrumentation - OpenAI wrapper
+ *
+ * @example
+ * ```typescript
+ * import { traceOpenAI } from '@ai-eval-platform/sdk/integrations/openai';
+ * import OpenAI from 'openai';
+ *
+ * const openai = new OpenAI({ apiKey: '...' });
+ * const tracedOpenAI = traceOpenAI(openai, client);
+ *
+ * // All calls are automatically traced
+ * const response = await tracedOpenAI.chat.completions.create({
+ *   model: 'gpt-4',
+ *   messages: [{ role: 'user', content: 'Hello!' }]
+ * });
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.traceOpenAI = traceOpenAI;
+exports.traceOpenAICall = traceOpenAICall;
+const context_1 = require("../context");
+/**
+ * Wrap OpenAI client with automatic tracing
+ *
+ * @example
+ * ```typescript
+ * import OpenAI from 'openai';
+ * import { traceOpenAI } from '@ai-eval-platform/sdk/integrations/openai';
+ *
+ * const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
+ * const tracedOpenAI = traceOpenAI(openai, evalClient);
+ *
+ * // Automatically traced
+ * const completion = await tracedOpenAI.chat.completions.create({
+ *   model: 'gpt-4',
+ *   messages: [{ role: 'user', content: 'Hello!' }]
+ * });
+ * ```
+ */
+function traceOpenAI(openai, evalClient, options = {}) {
+    const { captureInput = true, captureOutput = true, captureMetadata = true, organizationId, tracePrefix = "openai", } = options;
+    // Create proxy for chat.completions.create
+    const originalCreate = openai.chat.completions.create.bind(openai.chat.completions);
+    openai.chat.completions.create = async (params, requestOptions) => {
+        const startTime = Date.now();
+        const traceId = `${tracePrefix}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+        try {
+            // Call original method
+            const response = await originalCreate(params, requestOptions);
+            const durationMs = Date.now() - startTime;
+            // Create trace with success status and complete metadata
+            const traceMetadata = (0, context_1.mergeWithContext)({
+                model: params.model,
+                temperature: params.temperature,
+                max_tokens: params.max_tokens,
+                ...(captureInput ? { input: params.messages } : {}),
+                ...(captureOutput ? { output: response.choices[0]?.message } : {}),
+                ...(captureMetadata
+                    ? {
+                        usage: response.usage,
+                        finish_reason: response.choices[0]?.finish_reason,
+                    }
+                    : {}),
+            });
+            await evalClient.traces.create({
+                name: `OpenAI: ${params.model}`,
+                traceId,
+                organizationId: organizationId || evalClient.getOrganizationId(),
+                status: "success",
+                durationMs,
+                metadata: traceMetadata,
+            });
+            return response;
+        }
+        catch (error) {
+            const durationMs = Date.now() - startTime;
+            // Create trace with error status
+            const errorMetadata = (0, context_1.mergeWithContext)({
+                model: params.model,
+                temperature: params.temperature,
+                max_tokens: params.max_tokens,
+                ...(captureInput ? { input: params.messages } : {}),
+                ...(captureMetadata ? { params } : {}),
+                error: error instanceof Error ? error.message : String(error),
+            });
+            await evalClient.traces
+                .create({
+                name: `OpenAI: ${params.model}`,
+                traceId,
+                organizationId: organizationId || evalClient.getOrganizationId(),
+                status: "error",
+                durationMs,
+                metadata: errorMetadata,
+            })
+                .catch(() => {
+                // Ignore errors in trace creation to avoid masking the original error
+            });
+            throw error;
+        }
+    };
+    return openai;
+}
+/**
+ * Manual trace wrapper for OpenAI calls
+ *
+ * @example
+ * ```typescript
+ * const response = await traceOpenAICall(
+ *   evalClient,
+ *   'gpt-4-completion',
+ *   async () => {
+ *     return await openai.chat.completions.create({
+ *       model: 'gpt-4',
+ *       messages: [{ role: 'user', content: 'Hello!' }]
+ *     });
+ *   }
+ * );
+ * ```
+ */
+async function traceOpenAICall(evalClient, name, fn, options = {}) {
+    const startTime = Date.now();
+    const traceId = `openai-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+    try {
+        await evalClient.traces.create({
+            name,
+            traceId,
+            organizationId: options.organizationId || evalClient.getOrganizationId(),
+            status: "pending",
+            metadata: (0, context_1.mergeWithContext)({}),
+        });
+        const result = await fn();
+        const durationMs = Date.now() - startTime;
+        await evalClient.traces.create({
+            name,
+            traceId,
+            organizationId: options.organizationId || evalClient.getOrganizationId(),
+            status: "success",
+            durationMs,
+            metadata: (0, context_1.mergeWithContext)({}),
+        });
+        return result;
+    }
+    catch (error) {
+        const durationMs = Date.now() - startTime;
+        await evalClient.traces.create({
+            name,
+            traceId,
+            organizationId: options.organizationId || evalClient.getOrganizationId(),
+            status: "error",
+            durationMs,
+            metadata: (0, context_1.mergeWithContext)({
+                error: error instanceof Error ? error.message : String(error),
+            }),
+        });
+        throw error;
+    }
+}

package/dist/local.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Local Development Mode (Tier 2.10)
+ * Offline mode with local storage for development
+ *
+ * ⚠️ NOTE: This module requires Node.js and will not work in browsers.
+ */
+import type { Evaluation, Span, Trace } from "./types";
+export interface LocalStorageOptions {
+    directory?: string;
+    autoSave?: boolean;
+}
+export declare class LocalStorage {
+    private directory;
+    private autoSave;
+    private traces;
+    private evaluations;
+    private spans;
+    constructor(options?: LocalStorageOptions);
+    private initialize;
+    private loadAllData;
+    private saveTraceToDisk;
+    saveTrace(trace: Trace): Promise<void>;
+    getTrace(id: string): Promise<Trace | undefined>;
+    listTraces(): Promise<Trace[]>;
+    private saveEvaluationToDisk;
+    saveEvaluation(evaluation: Evaluation): Promise<void>;
+    getEvaluation(id: string): Promise<Evaluation | undefined>;
+    listEvaluations(): Promise<Evaluation[]>;
+    private saveSpansToDisk;
+    saveSpans(traceId: string, spans: Span[]): Promise<void>;
+    getSpans(traceId: string): Promise<Span[] | undefined>;
+    clear(): Promise<void>;
+    export(_format: "json"): Promise<string>;
+    getStats(): {
+        traces: number;
+        evaluations: number;
+        totalSpans: number;
+    };
+}

package/dist/local.js

@@ -0,0 +1,148 @@
+"use strict";
+/**
+ * Local Development Mode (Tier 2.10)
+ * Offline mode with local storage for development
+ *
+ * ⚠️ NOTE: This module requires Node.js and will not work in browsers.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalStorage = void 0;
+// Environment check
+const isNode = typeof process !== "undefined" && process.versions?.node;
+if (!isNode) {
+    throw new Error("Local storage mode requires Node.js and cannot run in browsers. " +
+        "This feature uses the filesystem for storing data.");
+}
+const promises_1 = __importDefault(require("node:fs/promises"));
+const node_path_1 = __importDefault(require("node:path"));
+class LocalStorage {
+    constructor(options = {}) {
+        this.traces = new Map();
+        this.evaluations = new Map();
+        this.spans = new Map();
+        this.directory = options.directory || "./.evalgate-data";
+        this.autoSave = options.autoSave !== false;
+        this.initialize();
+    }
+    async initialize() {
+        try {
+            await promises_1.default.mkdir(this.directory, { recursive: true });
+            await promises_1.default.mkdir(node_path_1.default.join(this.directory, "traces"), { recursive: true });
+            await promises_1.default.mkdir(node_path_1.default.join(this.directory, "evaluations"), {
+                recursive: true,
+            });
+            await promises_1.default.mkdir(node_path_1.default.join(this.directory, "spans"), { recursive: true });
+            // Load existing data
+            await this.loadAllData();
+        }
+        catch (error) {
+            console.warn("Failed to initialize local storage:", error);
+        }
+    }
+    async loadAllData() {
+        try {
+            // Load traces
+            const tracesDir = node_path_1.default.join(this.directory, "traces");
+            const traceFiles = await promises_1.default.readdir(tracesDir);
+            for (const file of traceFiles) {
+                if (file.endsWith(".json")) {
+                    const content = await promises_1.default.readFile(node_path_1.default.join(tracesDir, file), "utf-8");
+                    const trace = JSON.parse(content);
+                    this.traces.set(trace.id.toString(), trace);
+                }
+            }
+            // Load evaluations
+            const evalsDir = node_path_1.default.join(this.directory, "evaluations");
+            const evalFiles = await promises_1.default.readdir(evalsDir);
+            for (const file of evalFiles) {
+                if (file.endsWith(".json")) {
+                    const content = await promises_1.default.readFile(node_path_1.default.join(evalsDir, file), "utf-8");
+                    const evaluation = JSON.parse(content);
+                    this.evaluations.set(evaluation.id.toString(), evaluation);
+                }
+            }
+        }
+        catch (_error) {
+            // Directories might not exist yet, that's fine
+        }
+    }
+    async saveTraceToDisk(trace) {
+        const filePath = node_path_1.default.join(this.directory, "traces", `${trace.id}.json`);
+        await promises_1.default.writeFile(filePath, JSON.stringify(trace, null, 2));
+    }
+    async saveTrace(trace) {
+        this.traces.set(trace.id.toString(), trace);
+        if (this.autoSave) {
+            await this.saveTraceToDisk(trace);
+        }
+    }
+    async getTrace(id) {
+        return this.traces.get(id);
+    }
+    async listTraces() {
+        return Array.from(this.traces.values());
+    }
+    async saveEvaluationToDisk(evaluation) {
+        const filePath = node_path_1.default.join(this.directory, "evaluations", `${evaluation.id}.json`);
+        await promises_1.default.writeFile(filePath, JSON.stringify(evaluation, null, 2));
+    }
+    async saveEvaluation(evaluation) {
+        this.evaluations.set(evaluation.id.toString(), evaluation);
+        if (this.autoSave) {
+            await this.saveEvaluationToDisk(evaluation);
+        }
+    }
+    async getEvaluation(id) {
+        return this.evaluations.get(id);
+    }
+    async listEvaluations() {
+        return Array.from(this.evaluations.values());
+    }
+    async saveSpansToDisk(traceId, spans) {
+        const filePath = node_path_1.default.join(this.directory, "spans", `${traceId}.json`);
+        await promises_1.default.writeFile(filePath, JSON.stringify(spans, null, 2));
+    }
+    async saveSpans(traceId, spans) {
+        this.spans.set(traceId, spans);
+        if (this.autoSave) {
+            await this.saveSpansToDisk(traceId, spans);
+        }
+    }
+    async getSpans(traceId) {
+        return this.spans.get(traceId);
+    }
+    async clear() {
+        this.traces.clear();
+        this.evaluations.clear();
+        this.spans.clear();
+        // Optionally delete files
+        try {
+            await promises_1.default.rm(this.directory, { recursive: true, force: true });
+            await this.initialize();
+        }
+        catch (error) {
+            console.warn("Failed to clear local storage:", error);
+        }
+    }
+    async export(_format) {
+        const data = {
+            traces: Array.from(this.traces.values()),
+            evaluations: Array.from(this.evaluations.values()),
+            spans: Object.fromEntries(this.spans),
+        };
+        const exportPath = node_path_1.default.join(this.directory, `export-${Date.now()}.json`);
+        await promises_1.default.writeFile(exportPath, JSON.stringify(data, null, 2));
+        return exportPath;
+    }
+    getStats() {
+        return {
+            traces: this.traces.size,
+            evaluations: this.evaluations.size,
+            totalSpans: Array.from(this.spans.values()).reduce((sum, spans) => sum + spans.length, 0),
+        };
+    }
+}
+exports.LocalStorage = LocalStorage;

package/dist/logger.d.ts

@@ -0,0 +1,128 @@
+/**
+ * Debug Mode with Request Logging
+ * Tier 4.17: Troubleshooting utilities
+ *
+ * @example
+ * ```typescript
+ * import { createLogger } from '@ai-eval-platform/sdk';
+ *
+ * const logger = createLogger({ level: 'debug', pretty: true });
+ *
+ * logger.debug('Trace created', { traceId: '123' });
+ * logger.error('Request failed', { error: err });
+ * ```
+ */
+export type LogLevel = "trace" | "debug" | "info" | "warn" | "error";
+export interface LoggerOptions {
+    /** Log level (default: 'info') */
+    level?: LogLevel;
+    /** Pretty print logs (default: false) */
+    pretty?: boolean;
+    /** Include timestamps (default: true) */
+    timestamps?: boolean;
+    /** Custom log handler */
+    handler?: (entry: LogEntry) => void;
+    /** Prefix for all logs */
+    prefix?: string;
+}
+export interface LogEntry {
+    level: LogLevel;
+    message: string;
+    timestamp: string;
+    data?: unknown;
+    prefix?: string;
+}
+/**
+ * Logger for SDK debugging and troubleshooting
+ */
+export declare class Logger {
+    private options;
+    constructor(options?: LoggerOptions);
+    /**
+     * Log a trace message
+     */
+    trace(message: string, data?: unknown): void;
+    /**
+     * Log a debug message
+     */
+    debug(message: string, data?: unknown): void;
+    /**
+     * Log an info message
+     */
+    info(message: string, data?: unknown): void;
+    /**
+     * Log a warning message
+     */
+    warn(message: string, data?: unknown): void;
+    /**
+     * Log an error message
+     */
+    error(message: string, data?: unknown): void;
+    /**
+     * Log HTTP request
+     */
+    logRequest(method: string, url: string, data?: unknown): void;
+    /**
+     * Log HTTP response
+     */
+    logResponse(method: string, url: string, status: number, duration: number, data?: unknown): void;
+    /**
+     * Create child logger with prefix
+     */
+    child(prefix: string): Logger;
+    /**
+     * Set log level
+     */
+    setLevel(level: LogLevel): void;
+    /**
+     * Check if level is enabled
+     */
+    isLevelEnabled(level: LogLevel): boolean;
+    private log;
+    private defaultHandler;
+}
+/**
+ * Create a logger instance
+ *
+ * @example
+ * ```typescript
+ * const logger = createLogger({ level: 'debug', pretty: true });
+ * logger.debug('Starting evaluation');
+ * ```
+ */
+export declare function createLogger(options?: LoggerOptions): Logger;
+/**
+ * Get global logger (creates one if it doesn't exist)
+ */
+export declare function getLogger(): Logger;
+/**
+ * Set global logger
+ */
+export declare function setLogger(logger: Logger): void;
+/**
+ * Request/Response interceptor for logging
+ */
+export declare class RequestLogger {
+    private logger;
+    constructor(logger: Logger);
+    /**
+     * Log request before sending
+     */
+    logRequest(request: {
+        method: string;
+        url: string;
+        headers?: Record<string, string>;
+        body?: unknown;
+    }): void;
+    /**
+     * Log response after receiving
+     */
+    logResponse(response: {
+        method: string;
+        url: string;
+        status: number;
+        duration: number;
+        headers?: Record<string, string>;
+        body?: unknown;
+    }): void;
+}

package/dist/logger.js

@@ -0,0 +1,227 @@
+"use strict";
+/**
+ * Debug Mode with Request Logging
+ * Tier 4.17: Troubleshooting utilities
+ *
+ * @example
+ * ```typescript
+ * import { createLogger } from '@ai-eval-platform/sdk';
+ *
+ * const logger = createLogger({ level: 'debug', pretty: true });
+ *
+ * logger.debug('Trace created', { traceId: '123' });
+ * logger.error('Request failed', { error: err });
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RequestLogger = exports.Logger = void 0;
+exports.createLogger = createLogger;
+exports.getLogger = getLogger;
+exports.setLogger = setLogger;
+const LOG_LEVELS = {
+    trace: 0,
+    debug: 1,
+    info: 2,
+    warn: 3,
+    error: 4,
+};
+const LOG_COLORS = {
+    trace: "\x1b[90m", // gray
+    debug: "\x1b[36m", // cyan
+    info: "\x1b[32m", // green
+    warn: "\x1b[33m", // yellow
+    error: "\x1b[31m", // red
+};
+const COLOR_RESET = "\x1b[0m";
+/**
+ * Logger for SDK debugging and troubleshooting
+ */
+class Logger {
+    constructor(options = {}) {
+        this.options = {
+            level: options.level || "info",
+            pretty: options.pretty ?? false,
+            timestamps: options.timestamps ?? true,
+            handler: options.handler || this.defaultHandler.bind(this),
+            prefix: options.prefix || "EvalGate",
+        };
+    }
+    /**
+     * Log a trace message
+     */
+    trace(message, data) {
+        this.log("trace", message, data);
+    }
+    /**
+     * Log a debug message
+     */
+    debug(message, data) {
+        this.log("debug", message, data);
+    }
+    /**
+     * Log an info message
+     */
+    info(message, data) {
+        this.log("info", message, data);
+    }
+    /**
+     * Log a warning message
+     */
+    warn(message, data) {
+        this.log("warn", message, data);
+    }
+    /**
+     * Log an error message
+     */
+    error(message, data) {
+        this.log("error", message, data);
+    }
+    /**
+     * Log HTTP request
+     */
+    logRequest(method, url, data) {
+        this.debug(`→ ${method} ${url}`, data);
+    }
+    /**
+     * Log HTTP response
+     */
+    logResponse(method, url, status, duration, data) {
+        const level = status >= 400 ? "error" : status >= 300 ? "warn" : "debug";
+        this.log(level, `← ${method} ${url} ${status} (${duration}ms)`, data);
+    }
+    /**
+     * Create child logger with prefix
+     */
+    child(prefix) {
+        return new Logger({
+            ...this.options,
+            prefix: `${this.options.prefix}:${prefix}`,
+        });
+    }
+    /**
+     * Set log level
+     */
+    setLevel(level) {
+        this.options.level = level;
+    }
+    /**
+     * Check if level is enabled
+     */
+    isLevelEnabled(level) {
+        return LOG_LEVELS[level] >= LOG_LEVELS[this.options.level];
+    }
+    log(level, message, data) {
+        if (!this.isLevelEnabled(level))
+            return;
+        const entry = {
+            level,
+            message,
+            timestamp: new Date().toISOString(),
+            data,
+            prefix: this.options.prefix,
+        };
+        this.options.handler(entry);
+    }
+    defaultHandler(entry) {
+        const parts = [];
+        // Timestamp
+        if (this.options.timestamps) {
+            const time = this.options.pretty
+                ? new Date(entry.timestamp).toLocaleTimeString()
+                : entry.timestamp;
+            parts.push(this.options.pretty ? `\x1b[90m${time}${COLOR_RESET}` : time);
+        }
+        // Level
+        const levelStr = entry.level.toUpperCase().padEnd(5);
+        parts.push(this.options.pretty
+            ? `${LOG_COLORS[entry.level]}${levelStr}${COLOR_RESET}`
+            : levelStr);
+        // Prefix
+        if (entry.prefix) {
+            parts.push(this.options.pretty
+                ? `\x1b[35m[${entry.prefix}]${COLOR_RESET}`
+                : `[${entry.prefix}]`);
+        }
+        // Message
+        parts.push(entry.message);
+        // Log
+        const logLine = parts.join(" ");
+        if (entry.level === "error") {
+            console.error(logLine);
+        }
+        else if (entry.level === "warn") {
+            console.warn(logLine);
+        }
+        else {
+            console.log(logLine);
+        }
+        // Data
+        if (entry.data !== undefined) {
+            if (this.options.pretty) {
+                console.log(JSON.stringify(entry.data, null, 2));
+            }
+            else {
+                console.log(JSON.stringify(entry.data));
+            }
+        }
+    }
+}
+exports.Logger = Logger;
+/**
+ * Create a logger instance
+ *
+ * @example
+ * ```typescript
+ * const logger = createLogger({ level: 'debug', pretty: true });
+ * logger.debug('Starting evaluation');
+ * ```
+ */
+function createLogger(options) {
+    return new Logger(options);
+}
+/**
+ * Global logger instance
+ */
+let globalLogger;
+/**
+ * Get global logger (creates one if it doesn't exist)
+ */
+function getLogger() {
+    if (!globalLogger) {
+        globalLogger = new Logger();
+    }
+    return globalLogger;
+}
+/**
+ * Set global logger
+ */
+function setLogger(logger) {
+    globalLogger = logger;
+}
+/**
+ * Request/Response interceptor for logging
+ */
+class RequestLogger {
+    constructor(logger) {
+        this.logger = logger;
+    }
+    /**
+     * Log request before sending
+     */
+    logRequest(request) {
+        this.logger.logRequest(request.method, request.url, {
+            headers: request.headers,
+            body: request.body,
+        });
+    }
+    /**
+     * Log response after receiving
+     */
+    logResponse(response) {
+        this.logger.logResponse(response.method, response.url, response.status, response.duration, {
+            headers: response.headers,
+            body: response.body,
+        });
+    }
+}
+exports.RequestLogger = RequestLogger;

package/dist/matchers/index.d.ts

@@ -0,0 +1 @@
+export { extendExpectWithToPassGate, toPassGate } from "./to-pass-gate";