@pauly4010/evalai-sdk 1.3.0

package/dist/integrations/anthropic.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Anthropic Integration
+ * Tier 1.2: Framework Auto-Instrumentation - Anthropic wrapper
+ *
+ * @example
+ * ```typescript
+ * import { traceAnthropic } from '@ai-eval-platform/sdk/integrations/anthropic';
+ * import Anthropic from '@anthropic-ai/sdk';
+ *
+ * const anthropic = new Anthropic({ apiKey: '...' });
+ * const tracedAnthropic = traceAnthropic(anthropic, client);
+ *
+ * // All calls are automatically traced
+ * const message = await tracedAnthropic.messages.create({
+ *   model: 'claude-3-5-sonnet-20241022',
+ *   max_tokens: 1024,
+ *   messages: [{ role: 'user', content: 'Hello!' }]
+ * });
+ * ```
+ */
+import type { AIEvalClient } from '../client';
+export interface AnthropicTraceOptions {
+    /** Whether to capture input (default: true) */
+    captureInput?: boolean;
+    /** Whether to capture output (default: true) */
+    captureOutput?: boolean;
+    /** Whether to capture metadata (default: true) */
+    captureMetadata?: boolean;
+    /** Organization ID for traces */
+    organizationId?: number;
+    /** Custom trace name prefix */
+    tracePrefix?: string;
+}
+/**
+ * Wrap Anthropic client with automatic tracing
+ *
+ * @example
+ * ```typescript
+ * import Anthropic from '@anthropic-ai/sdk';
+ * import { traceAnthropic } from '@ai-eval-platform/sdk/integrations/anthropic';
+ *
+ * const anthropic = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
+ * const tracedAnthropic = traceAnthropic(anthropic, evalClient);
+ *
+ * // Automatically traced
+ * const message = await tracedAnthropic.messages.create({
+ *   model: 'claude-3-5-sonnet-20241022',
+ *   max_tokens: 1024,
+ *   messages: [{ role: 'user', content: 'Hello, Claude!' }]
+ * });
+ * ```
+ */
+export declare function traceAnthropic(anthropic: any, evalClient: AIEvalClient, options?: AnthropicTraceOptions): any;
+/**
+ * Manual trace wrapper for Anthropic calls
+ *
+ * @example
+ * ```typescript
+ * const message = await traceAnthropicCall(
+ *   evalClient,
+ *   'claude-completion',
+ *   async () => {
+ *     return await anthropic.messages.create({
+ *       model: 'claude-3-5-sonnet-20241022',
+ *       max_tokens: 1024,
+ *       messages: [{ role: 'user', content: 'Hello!' }]
+ *     });
+ *   }
+ * );
+ * ```
+ */
+export declare function traceAnthropicCall<T>(evalClient: AIEvalClient, name: string, fn: () => Promise<T>, options?: AnthropicTraceOptions): Promise<T>;

package/dist/integrations/anthropic.js

@@ -0,0 +1,159 @@
+"use strict";
+/**
+ * Anthropic Integration
+ * Tier 1.2: Framework Auto-Instrumentation - Anthropic wrapper
+ *
+ * @example
+ * ```typescript
+ * import { traceAnthropic } from '@ai-eval-platform/sdk/integrations/anthropic';
+ * import Anthropic from '@anthropic-ai/sdk';
+ *
+ * const anthropic = new Anthropic({ apiKey: '...' });
+ * const tracedAnthropic = traceAnthropic(anthropic, client);
+ *
+ * // All calls are automatically traced
+ * const message = await tracedAnthropic.messages.create({
+ *   model: 'claude-3-5-sonnet-20241022',
+ *   max_tokens: 1024,
+ *   messages: [{ role: 'user', content: 'Hello!' }]
+ * });
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.traceAnthropic = traceAnthropic;
+exports.traceAnthropicCall = traceAnthropicCall;
+const context_1 = require("../context");
+/**
+ * Wrap Anthropic client with automatic tracing
+ *
+ * @example
+ * ```typescript
+ * import Anthropic from '@anthropic-ai/sdk';
+ * import { traceAnthropic } from '@ai-eval-platform/sdk/integrations/anthropic';
+ *
+ * const anthropic = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
+ * const tracedAnthropic = traceAnthropic(anthropic, evalClient);
+ *
+ * // Automatically traced
+ * const message = await tracedAnthropic.messages.create({
+ *   model: 'claude-3-5-sonnet-20241022',
+ *   max_tokens: 1024,
+ *   messages: [{ role: 'user', content: 'Hello, Claude!' }]
+ * });
+ * ```
+ */
+function traceAnthropic(anthropic, evalClient, options = {}) {
+    const { captureInput = true, captureOutput = true, captureMetadata = true, organizationId, tracePrefix = 'anthropic' } = options;
+    // Create proxy for messages.create
+    const originalCreate = anthropic.messages.create.bind(anthropic.messages);
+    anthropic.messages.create = async (params, requestOptions) => {
+        const startTime = Date.now();
+        const traceId = `${tracePrefix}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+        try {
+            // Call original method
+            const message = await originalCreate(params, requestOptions);
+            const durationMs = Date.now() - startTime;
+            // Create trace with success status and complete metadata
+            const traceMetadata = (0, context_1.mergeWithContext)({
+                model: params.model,
+                temperature: params.temperature,
+                max_tokens: params.max_tokens,
+                ...(captureInput ? { input: params.messages } : {}),
+                ...(captureOutput ? { output: message.content } : {}),
+                ...(captureMetadata ? {
+                    usage: message.usage,
+                    stop_reason: message.stop_reason
+                } : {})
+            });
+            await evalClient.traces.create({
+                name: `Anthropic: ${params.model}`,
+                traceId,
+                organizationId: organizationId || evalClient.getOrganizationId(),
+                status: 'success',
+                durationMs,
+                metadata: traceMetadata
+            });
+            return message;
+        }
+        catch (error) {
+            const durationMs = Date.now() - startTime;
+            // Create trace with error status
+            const errorMetadata = (0, context_1.mergeWithContext)({
+                model: params.model,
+                temperature: params.temperature,
+                max_tokens: params.max_tokens,
+                ...(captureInput ? { input: params.messages } : {}),
+                ...(captureMetadata ? { params } : {}),
+                error: error instanceof Error ? error.message : String(error)
+            });
+            await evalClient.traces.create({
+                name: `Anthropic: ${params.model}`,
+                traceId,
+                organizationId: organizationId || evalClient.getOrganizationId(),
+                status: 'error',
+                durationMs,
+                metadata: errorMetadata
+            }).catch(() => {
+                // Ignore errors in trace creation to avoid masking the original error
+            });
+            throw error;
+        }
+    };
+    return anthropic;
+}
+/**
+ * Manual trace wrapper for Anthropic calls
+ *
+ * @example
+ * ```typescript
+ * const message = await traceAnthropicCall(
+ *   evalClient,
+ *   'claude-completion',
+ *   async () => {
+ *     return await anthropic.messages.create({
+ *       model: 'claude-3-5-sonnet-20241022',
+ *       max_tokens: 1024,
+ *       messages: [{ role: 'user', content: 'Hello!' }]
+ *     });
+ *   }
+ * );
+ * ```
+ */
+async function traceAnthropicCall(evalClient, name, fn, options = {}) {
+    const startTime = Date.now();
+    const traceId = `anthropic-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+    try {
+        await evalClient.traces.create({
+            name,
+            traceId,
+            organizationId: options.organizationId || evalClient.getOrganizationId(),
+            status: 'pending',
+            metadata: (0, context_1.mergeWithContext)({})
+        });
+        const result = await fn();
+        const durationMs = Date.now() - startTime;
+        await evalClient.traces.create({
+            name,
+            traceId,
+            organizationId: options.organizationId || evalClient.getOrganizationId(),
+            status: 'success',
+            durationMs,
+            metadata: (0, context_1.mergeWithContext)({})
+        });
+        return result;
+    }
+    catch (error) {
+        const durationMs = Date.now() - startTime;
+        await evalClient.traces.create({
+            name,
+            traceId,
+            organizationId: options.organizationId || evalClient.getOrganizationId(),
+            status: 'error',
+            durationMs,
+            metadata: (0, context_1.mergeWithContext)({
+                error: error instanceof Error ? error.message : String(error)
+            })
+        });
+        throw error;
+    }
+}

package/dist/integrations/openai.d.ts

@@ -0,0 +1,69 @@
+/**
+ * OpenAI Integration
+ * Tier 1.2: Framework Auto-Instrumentation - OpenAI wrapper
+ *
+ * @example
+ * ```typescript
+ * import { traceOpenAI } from '@ai-eval-platform/sdk/integrations/openai';
+ * import OpenAI from 'openai';
+ *
+ * const openai = new OpenAI({ apiKey: '...' });
+ * const tracedOpenAI = traceOpenAI(openai, client);
+ *
+ * // All calls are automatically traced
+ * const response = await tracedOpenAI.chat.completions.create({
+ *   model: 'gpt-4',
+ *   messages: [{ role: 'user', content: 'Hello!' }]
+ * });
+ * ```
+ */
+import type { AIEvalClient } from '../client';
+export interface OpenAITraceOptions {
+    /** Whether to capture input (default: true) */
+    captureInput?: boolean;
+    /** Whether to capture output (default: true) */
+    captureOutput?: boolean;
+    /** Whether to capture metadata (default: true) */
+    captureMetadata?: boolean;
+    /** Organization ID for traces */
+    organizationId?: number;
+    /** Custom trace name prefix */
+    tracePrefix?: string;
+}
+/**
+ * Wrap OpenAI client with automatic tracing
+ *
+ * @example
+ * ```typescript
+ * import OpenAI from 'openai';
+ * import { traceOpenAI } from '@ai-eval-platform/sdk/integrations/openai';
+ *
+ * const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
+ * const tracedOpenAI = traceOpenAI(openai, evalClient);
+ *
+ * // Automatically traced
+ * const completion = await tracedOpenAI.chat.completions.create({
+ *   model: 'gpt-4',
+ *   messages: [{ role: 'user', content: 'Hello!' }]
+ * });
+ * ```
+ */
+export declare function traceOpenAI(openai: any, evalClient: AIEvalClient, options?: OpenAITraceOptions): any;
+/**
+ * Manual trace wrapper for OpenAI calls
+ *
+ * @example
+ * ```typescript
+ * const response = await traceOpenAICall(
+ *   evalClient,
+ *   'gpt-4-completion',
+ *   async () => {
+ *     return await openai.chat.completions.create({
+ *       model: 'gpt-4',
+ *       messages: [{ role: 'user', content: 'Hello!' }]
+ *     });
+ *   }
+ * );
+ * ```
+ */
+export declare function traceOpenAICall<T>(evalClient: AIEvalClient, name: string, fn: () => Promise<T>, options?: OpenAITraceOptions): Promise<T>;

package/dist/integrations/openai.js

@@ -0,0 +1,156 @@
+"use strict";
+/**
+ * OpenAI Integration
+ * Tier 1.2: Framework Auto-Instrumentation - OpenAI wrapper
+ *
+ * @example
+ * ```typescript
+ * import { traceOpenAI } from '@ai-eval-platform/sdk/integrations/openai';
+ * import OpenAI from 'openai';
+ *
+ * const openai = new OpenAI({ apiKey: '...' });
+ * const tracedOpenAI = traceOpenAI(openai, client);
+ *
+ * // All calls are automatically traced
+ * const response = await tracedOpenAI.chat.completions.create({
+ *   model: 'gpt-4',
+ *   messages: [{ role: 'user', content: 'Hello!' }]
+ * });
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.traceOpenAI = traceOpenAI;
+exports.traceOpenAICall = traceOpenAICall;
+const context_1 = require("../context");
+/**
+ * Wrap OpenAI client with automatic tracing
+ *
+ * @example
+ * ```typescript
+ * import OpenAI from 'openai';
+ * import { traceOpenAI } from '@ai-eval-platform/sdk/integrations/openai';
+ *
+ * const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
+ * const tracedOpenAI = traceOpenAI(openai, evalClient);
+ *
+ * // Automatically traced
+ * const completion = await tracedOpenAI.chat.completions.create({
+ *   model: 'gpt-4',
+ *   messages: [{ role: 'user', content: 'Hello!' }]
+ * });
+ * ```
+ */
+function traceOpenAI(openai, evalClient, options = {}) {
+    const { captureInput = true, captureOutput = true, captureMetadata = true, organizationId, tracePrefix = 'openai' } = options;
+    // Create proxy for chat.completions.create
+    const originalCreate = openai.chat.completions.create.bind(openai.chat.completions);
+    openai.chat.completions.create = async (params, requestOptions) => {
+        const startTime = Date.now();
+        const traceId = `${tracePrefix}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+        try {
+            // Call original method
+            const response = await originalCreate(params, requestOptions);
+            const durationMs = Date.now() - startTime;
+            // Create trace with success status and complete metadata
+            const traceMetadata = (0, context_1.mergeWithContext)({
+                model: params.model,
+                temperature: params.temperature,
+                max_tokens: params.max_tokens,
+                ...(captureInput ? { input: params.messages } : {}),
+                ...(captureOutput ? { output: response.choices[0]?.message } : {}),
+                ...(captureMetadata ? {
+                    usage: response.usage,
+                    finish_reason: response.choices[0]?.finish_reason
+                } : {})
+            });
+            await evalClient.traces.create({
+                name: `OpenAI: ${params.model}`,
+                traceId,
+                organizationId: organizationId || evalClient.getOrganizationId(),
+                status: 'success',
+                durationMs,
+                metadata: traceMetadata
+            });
+            return response;
+        }
+        catch (error) {
+            const durationMs = Date.now() - startTime;
+            // Create trace with error status
+            const errorMetadata = (0, context_1.mergeWithContext)({
+                model: params.model,
+                temperature: params.temperature,
+                max_tokens: params.max_tokens,
+                ...(captureInput ? { input: params.messages } : {}),
+                ...(captureMetadata ? { params } : {}),
+                error: error instanceof Error ? error.message : String(error)
+            });
+            await evalClient.traces.create({
+                name: `OpenAI: ${params.model}`,
+                traceId,
+                organizationId: organizationId || evalClient.getOrganizationId(),
+                status: 'error',
+                durationMs,
+                metadata: errorMetadata
+            }).catch(() => {
+                // Ignore errors in trace creation to avoid masking the original error
+            });
+            throw error;
+        }
+    };
+    return openai;
+}
+/**
+ * Manual trace wrapper for OpenAI calls
+ *
+ * @example
+ * ```typescript
+ * const response = await traceOpenAICall(
+ *   evalClient,
+ *   'gpt-4-completion',
+ *   async () => {
+ *     return await openai.chat.completions.create({
+ *       model: 'gpt-4',
+ *       messages: [{ role: 'user', content: 'Hello!' }]
+ *     });
+ *   }
+ * );
+ * ```
+ */
+async function traceOpenAICall(evalClient, name, fn, options = {}) {
+    const startTime = Date.now();
+    const traceId = `openai-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+    try {
+        await evalClient.traces.create({
+            name,
+            traceId,
+            organizationId: options.organizationId || evalClient.getOrganizationId(),
+            status: 'pending',
+            metadata: (0, context_1.mergeWithContext)({})
+        });
+        const result = await fn();
+        const durationMs = Date.now() - startTime;
+        await evalClient.traces.create({
+            name,
+            traceId,
+            organizationId: options.organizationId || evalClient.getOrganizationId(),
+            status: 'success',
+            durationMs,
+            metadata: (0, context_1.mergeWithContext)({})
+        });
+        return result;
+    }
+    catch (error) {
+        const durationMs = Date.now() - startTime;
+        await evalClient.traces.create({
+            name,
+            traceId,
+            organizationId: options.organizationId || evalClient.getOrganizationId(),
+            status: 'error',
+            durationMs,
+            metadata: (0, context_1.mergeWithContext)({
+                error: error instanceof Error ? error.message : String(error)
+            })
+        });
+        throw error;
+    }
+}

package/dist/local.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Local Development Mode (Tier 2.10)
+ * Offline mode with local storage for development
+ *
+ * ⚠️ NOTE: This module requires Node.js and will not work in browsers.
+ */
+import type { Trace, Evaluation, Span } from './types';
+export interface LocalStorageOptions {
+    directory?: string;
+    autoSave?: boolean;
+}
+export declare class LocalStorage {
+    private directory;
+    private autoSave;
+    private traces;
+    private evaluations;
+    private spans;
+    constructor(options?: LocalStorageOptions);
+    private initialize;
+    private loadAllData;
+    private saveTraceToDisk;
+    saveTrace(trace: Trace): Promise<void>;
+    getTrace(id: string): Promise<Trace | undefined>;
+    listTraces(): Promise<Trace[]>;
+    private saveEvaluationToDisk;
+    saveEvaluation(evaluation: Evaluation): Promise<void>;
+    getEvaluation(id: string): Promise<Evaluation | undefined>;
+    listEvaluations(): Promise<Evaluation[]>;
+    private saveSpansToDisk;
+    saveSpans(traceId: string, spans: Span[]): Promise<void>;
+    getSpans(traceId: string): Promise<Span[] | undefined>;
+    clear(): Promise<void>;
+    export(format: 'json'): Promise<string>;
+    getStats(): {
+        traces: number;
+        evaluations: number;
+        totalSpans: number;
+    };
+}

package/dist/local.js

@@ -0,0 +1,146 @@
+"use strict";
+/**
+ * Local Development Mode (Tier 2.10)
+ * Offline mode with local storage for development
+ *
+ * ⚠️ NOTE: This module requires Node.js and will not work in browsers.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalStorage = void 0;
+// Environment check
+const isNode = typeof process !== 'undefined' && process.versions?.node;
+if (!isNode) {
+    throw new Error('Local storage mode requires Node.js and cannot run in browsers. ' +
+        'This feature uses the filesystem for storing data.');
+}
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+class LocalStorage {
+    constructor(options = {}) {
+        this.traces = new Map();
+        this.evaluations = new Map();
+        this.spans = new Map();
+        this.directory = options.directory || './.evalai-data';
+        this.autoSave = options.autoSave !== false;
+        this.initialize();
+    }
+    async initialize() {
+        try {
+            await promises_1.default.mkdir(this.directory, { recursive: true });
+            await promises_1.default.mkdir(path_1.default.join(this.directory, 'traces'), { recursive: true });
+            await promises_1.default.mkdir(path_1.default.join(this.directory, 'evaluations'), { recursive: true });
+            await promises_1.default.mkdir(path_1.default.join(this.directory, 'spans'), { recursive: true });
+            // Load existing data
+            await this.loadAllData();
+        }
+        catch (error) {
+            console.warn('Failed to initialize local storage:', error);
+        }
+    }
+    async loadAllData() {
+        try {
+            // Load traces
+            const tracesDir = path_1.default.join(this.directory, 'traces');
+            const traceFiles = await promises_1.default.readdir(tracesDir);
+            for (const file of traceFiles) {
+                if (file.endsWith('.json')) {
+                    const content = await promises_1.default.readFile(path_1.default.join(tracesDir, file), 'utf-8');
+                    const trace = JSON.parse(content);
+                    this.traces.set(trace.id.toString(), trace);
+                }
+            }
+            // Load evaluations
+            const evalsDir = path_1.default.join(this.directory, 'evaluations');
+            const evalFiles = await promises_1.default.readdir(evalsDir);
+            for (const file of evalFiles) {
+                if (file.endsWith('.json')) {
+                    const content = await promises_1.default.readFile(path_1.default.join(evalsDir, file), 'utf-8');
+                    const evaluation = JSON.parse(content);
+                    this.evaluations.set(evaluation.id.toString(), evaluation);
+                }
+            }
+        }
+        catch (error) {
+            // Directories might not exist yet, that's fine
+        }
+    }
+    async saveTraceToDisk(trace) {
+        const filePath = path_1.default.join(this.directory, 'traces', `${trace.id}.json`);
+        await promises_1.default.writeFile(filePath, JSON.stringify(trace, null, 2));
+    }
+    async saveTrace(trace) {
+        this.traces.set(trace.id.toString(), trace);
+        if (this.autoSave) {
+            await this.saveTraceToDisk(trace);
+        }
+    }
+    async getTrace(id) {
+        return this.traces.get(id);
+    }
+    async listTraces() {
+        return Array.from(this.traces.values());
+    }
+    async saveEvaluationToDisk(evaluation) {
+        const filePath = path_1.default.join(this.directory, 'evaluations', `${evaluation.id}.json`);
+        await promises_1.default.writeFile(filePath, JSON.stringify(evaluation, null, 2));
+    }
+    async saveEvaluation(evaluation) {
+        this.evaluations.set(evaluation.id.toString(), evaluation);
+        if (this.autoSave) {
+            await this.saveEvaluationToDisk(evaluation);
+        }
+    }
+    async getEvaluation(id) {
+        return this.evaluations.get(id);
+    }
+    async listEvaluations() {
+        return Array.from(this.evaluations.values());
+    }
+    async saveSpansToDisk(traceId, spans) {
+        const filePath = path_1.default.join(this.directory, 'spans', `${traceId}.json`);
+        await promises_1.default.writeFile(filePath, JSON.stringify(spans, null, 2));
+    }
+    async saveSpans(traceId, spans) {
+        this.spans.set(traceId, spans);
+        if (this.autoSave) {
+            await this.saveSpansToDisk(traceId, spans);
+        }
+    }
+    async getSpans(traceId) {
+        return this.spans.get(traceId);
+    }
+    async clear() {
+        this.traces.clear();
+        this.evaluations.clear();
+        this.spans.clear();
+        // Optionally delete files
+        try {
+            await promises_1.default.rm(this.directory, { recursive: true, force: true });
+            await this.initialize();
+        }
+        catch (error) {
+            console.warn('Failed to clear local storage:', error);
+        }
+    }
+    async export(format) {
+        const data = {
+            traces: Array.from(this.traces.values()),
+            evaluations: Array.from(this.evaluations.values()),
+            spans: Object.fromEntries(this.spans)
+        };
+        const exportPath = path_1.default.join(this.directory, `export-${Date.now()}.json`);
+        await promises_1.default.writeFile(exportPath, JSON.stringify(data, null, 2));
+        return exportPath;
+    }
+    getStats() {
+        return {
+            traces: this.traces.size,
+            evaluations: this.evaluations.size,
+            totalSpans: Array.from(this.spans.values()).reduce((sum, spans) => sum + spans.length, 0)
+        };
+    }
+}
+exports.LocalStorage = LocalStorage;