@axon-ai/openai-tracer 1.0.3

package/README.md

@@ -0,0 +1,276 @@
+# OpenAI Function Calling Tracer
+
+A comprehensive tracing solution for OpenAI Function Calling agents, providing detailed monitoring, cost analysis, and performance insights.
+
+## 🚀 Features
+
+- **Function Call Tracking**: Monitor all function calls with detailed parameters and results
+- **Tool Selection Analysis**: Track which tools are selected and why
+- **Cost Calculation**: Automatic cost calculation based on token usage and model pricing
+- **Performance Metrics**: Latency tracking and performance analysis
+- **Error Monitoring**: Comprehensive error tracking and debugging
+- **Real-time Dashboard**: Live visualization of agent execution
+- **Conversation Flow**: Track multi-turn conversations and context
+
+## 📦 Installation
+
+```bash
+npm install @axon-ai/openai-tracer openai
+```
+
+## 🎯 Quick Start
+
+### Basic Usage
+
+```javascript
+import OpenAI from 'openai';
+import { createOpenAITracer, TracedOpenAI } from '@axon-ai/openai-tracer';
+
+// Initialize OpenAI client
+const openai = new OpenAI({
+  apiKey: process.env.OPENAI_API_KEY,
+});
+
+// Create tracer
+const tracer = createOpenAITracer({
+  projectName: 'my-agent',
+  metadata: {
+    version: '1.0.0',
+    environment: 'production',
+  },
+});
+
+// Create traced OpenAI client
+const tracedOpenAI = new TracedOpenAI(openai, tracer);
+
+// Use the traced client
+const response = await tracedOpenAI.createChatCompletion({
+  model: 'gpt-4',
+  messages: [
+    { role: 'user', content: 'What\'s the weather in NYC?' }
+  ],
+  tools: [/* your tools */],
+});
+```
+
+### Advanced Configuration
+
+```javascript
+const tracer = createOpenAITracer({
+  projectName: 'weather-agent',
+  endpoint: 'http://localhost:3000', // Custom trace server
+  metadata: {
+    agentType: 'weather',
+    version: '2.1.0',
+    environment: 'staging',
+    team: 'ai-platform',
+  },
+  autoConnect: true, // Auto-connect to trace server
+});
+```
+
+## 🔧 API Reference
+
+### OpenAITracer
+
+#### Constructor Options
+
+```typescript
+interface OpenAITraceConfig {
+  projectName?: string;        // Project identifier
+  endpoint?: string;          // Trace server endpoint
+  metadata?: Record<string, any>; // Custom metadata
+  autoConnect?: boolean;      // Auto-connect to server
+}
+```
+
+#### Methods
+
+##### `traceFunctionCallStart(functionName, arguments, model, messages, tools?)`
+Track the start of a function call.
+
+##### `traceFunctionCallEnd(eventId, result, cost, latency, tokens?)`
+Track the completion of a function call.
+
+##### `traceToolSelection(availableTools, selectedTool, reasoning?, confidence?)`
+Track tool selection decisions.
+
+##### `traceConversationTurn(userMessage, assistantResponse, model, tokens?, cost?)`
+Track conversation turns.
+
+##### `traceError(error, context, functionName?, arguments?)`
+Track errors and exceptions.
+
+### TracedOpenAI
+
+A wrapper around the OpenAI client that automatically traces all interactions.
+
+#### Methods
+
+##### `createChatCompletion(params)`
+Enhanced chat completion with automatic tracing.
+
+## 📊 Dashboard Integration
+
+The tracer automatically sends data to the Agent Trace dashboard for visualization:
+
+- **Function Call Flow**: Visual representation of function calls
+- **Cost Analysis**: Detailed cost breakdown by function and model
+- **Performance Metrics**: Latency and throughput analysis
+- **Tool Usage**: Which tools are used most frequently
+- **Error Tracking**: Error rates and debugging information
+
+## 🎨 Example Agents
+
+### Weather Agent
+
+```javascript
+import { createOpenAITracer, TracedOpenAI } from '@axon-ai/openai-tracer';
+
+const tracer = createOpenAITracer({ projectName: 'weather-agent' });
+const tracedOpenAI = new TracedOpenAI(openai, tracer);
+
+const tools = [
+  {
+    type: 'function',
+    function: {
+      name: 'getCurrentWeather',
+      description: 'Get current weather',
+      parameters: {
+        type: 'object',
+        properties: {
+          location: { type: 'string' }
+        }
+      }
+    }
+  }
+];
+
+const response = await tracedOpenAI.createChatCompletion({
+  model: 'gpt-4',
+  messages: [{ role: 'user', content: 'Weather in NYC?' }],
+  tools,
+});
+```
+
+### Stock Analysis Agent
+
+```javascript
+const tracer = createOpenAITracer({ projectName: 'stock-agent' });
+const tracedOpenAI = new TracedOpenAI(openai, tracer);
+
+const tools = [
+  {
+    type: 'function',
+    function: {
+      name: 'getStockPrice',
+      description: 'Get stock price',
+      parameters: {
+        type: 'object',
+        properties: {
+          symbol: { type: 'string' }
+        }
+      }
+    }
+  }
+];
+
+const response = await tracedOpenAI.createChatCompletion({
+  model: 'gpt-4',
+  messages: [{ role: 'user', content: 'AAPL stock price?' }],
+  tools,
+});
+```
+
+## 🔍 Event Types
+
+The tracer tracks several types of events:
+
+### Function Call Events
+- `function_call_start`: When a function call begins
+- `function_call_end`: When a function call completes
+
+### Tool Selection Events
+- `tool_selection`: When the model selects a tool
+
+### Conversation Events
+- `conversation_turn`: Each user-assistant interaction
+
+### Error Events
+- `error`: When errors occur during execution
+
+## 💰 Cost Calculation
+
+Automatic cost calculation based on:
+
+- **Model Pricing**: Current OpenAI pricing for different models
+- **Token Usage**: Prompt and completion tokens
+- **Function Calls**: Additional costs for function calling
+
+Supported models:
+- GPT-4: $0.03/1K prompt, $0.06/1K completion
+- GPT-4 Turbo: $0.01/1K prompt, $0.03/1K completion
+- GPT-3.5 Turbo: $0.001/1K prompt, $0.002/1K completion
+
+## 🚨 Error Handling
+
+The tracer provides comprehensive error tracking:
+
+```javascript
+try {
+  const response = await tracedOpenAI.createChatCompletion(params);
+} catch (error) {
+  // Error is automatically traced
+  console.error('API call failed:', error);
+}
+```
+
+## 🔧 Development
+
+### Building
+
+```bash
+npm run build
+```
+
+### Development Mode
+
+```bash
+npm run dev
+```
+
+### Testing
+
+```bash
+npm test
+```
+
+## 📈 Performance
+
+The tracer is designed for minimal overhead:
+
+- **Async Operations**: Non-blocking event queuing
+- **Batch Processing**: Efficient event batching
+- **Memory Management**: Automatic cleanup of old events
+- **Network Optimization**: Compressed data transmission
+
+## 🤝 Contributing
+
+1. Fork the repository
+2. Create a feature branch
+3. Make your changes
+4. Add tests
+5. Submit a pull request
+
+## 📄 License
+
+MIT License - see LICENSE file for details.
+
+## 🔄 Changelog
+
+### v1.0.0
+- Initial release
+- OpenAI Function Calling support
+- Cost calculation
+- Performance metrics
+- Dashboard integration

package/dist/index.d.mts

@@ -0,0 +1,214 @@
+import { EventEmitter } from 'events';
+
+/**
+ * OpenAI Function Calling Tracer
+ * Tracks OpenAI function calling agents with detailed event logging
+ */
+
+interface OpenAIFunctionCall {
+    id: string;
+    type: 'function';
+    function: {
+        name: string;
+        arguments: string;
+    };
+}
+interface OpenAIToolCall {
+    id: string;
+    type: 'function';
+    function: {
+        name: string;
+        arguments: string;
+    };
+}
+interface OpenAIMessage {
+    role: 'system' | 'user' | 'assistant' | 'tool';
+    content: string | null;
+    tool_calls?: OpenAIToolCall[];
+    tool_call_id?: string;
+    name?: string;
+}
+interface OpenAITool {
+    type: 'function';
+    function: {
+        name: string;
+        description: string;
+        parameters: any;
+    };
+}
+interface OpenAITraceEvent {
+    eventId: string;
+    traceId: string;
+    timestamp: number;
+    type: 'function_call_start' | 'function_call_end' | 'tool_selection' | 'conversation_turn' | 'error';
+    data: any;
+    metadata?: Record<string, any>;
+}
+interface OpenAITraceConfig {
+    projectName?: string;
+    endpoint?: string;
+    metadata?: Record<string, any>;
+    autoConnect?: boolean;
+}
+/**
+ * OpenAI Function Calling Tracer
+ *
+ * This class provides comprehensive tracing capabilities for OpenAI function calling agents.
+ * It tracks function calls, tool selections, conversation turns, and errors, sending
+ * real-time events to the Agent Trace Visualizer backend via WebSocket.
+ *
+ * Features:
+ * - Real-time event queuing and flushing
+ * - WebSocket connection management
+ * - Cost calculation for different OpenAI models
+ * - Error tracking and reporting
+ * - Automatic reconnection handling
+ */
+declare class OpenAITracer extends EventEmitter {
+    private traceId;
+    private projectName;
+    private endpoint;
+    private metadata;
+    private eventQueue;
+    private isConnected;
+    private client;
+    private flushInterval;
+    /**
+     * Creates a new OpenAI tracer instance
+     *
+     * @param config - Configuration object for the tracer
+     * @param config.projectName - Project name for organizing traces (default: 'openai-agent')
+     * @param config.endpoint - Backend server endpoint (default: 'http://localhost:3000')
+     * @param config.metadata - Additional metadata to include with all events
+     * @param config.autoConnect - Whether to automatically connect to the server (default: true)
+     */
+    constructor(config?: OpenAITraceConfig);
+    /**
+     * Establishes WebSocket connection to the trace server
+     *
+     * This method creates a Socket.IO client connection to the backend server,
+     * sets up event listeners for connection status, and starts the periodic
+     * event flushing mechanism.
+     *
+     * @throws Error if connection fails
+     */
+    connect(): Promise<void>;
+    /**
+     * Disconnect from the trace server
+     */
+    disconnect(): void;
+    /**
+     * Records the start of a function call execution
+     *
+     * This method creates and queues a function call start event, capturing
+     * the function name, arguments, model, messages, and available tools.
+     * It returns an event ID that can be used to correlate with the end event.
+     *
+     * @param functionName - Name of the function being called
+     * @param functionArguments - Arguments passed to the function
+     * @param model - OpenAI model being used (e.g., 'gpt-4', 'gpt-3.5-turbo')
+     * @param messages - Array of conversation messages
+     * @param tools - Optional array of available tools
+     * @returns Event ID for correlating with the corresponding end event
+     */
+    traceFunctionCallStart(functionName: string, functionArguments: any, model: string, messages: OpenAIMessage[], tools?: OpenAITool[]): string;
+    /**
+     * Track function call end
+     */
+    traceFunctionCallEnd(eventId: string, result: any, cost: number, latency: number, tokens?: {
+        prompt: number;
+        completion: number;
+        total: number;
+    }): void;
+    /**
+     * Track tool selection
+     */
+    traceToolSelection(availableTools: OpenAITool[], selectedTool: OpenAITool, reasoning?: string, confidence?: number): void;
+    /**
+     * Track conversation turn
+     */
+    traceConversationTurn(userMessage: string, assistantResponse: string, model: string, tokens?: {
+        prompt: number;
+        completion: number;
+        total: number;
+    }, cost?: number): void;
+    /**
+     * Track error
+     */
+    traceError(error: Error, context: string, functionName?: string, functionArguments?: any): void;
+    /**
+     * Add event to queue
+     */
+    private addEvent;
+    /**
+     * Flush events to server
+     */
+    flushQueue(): Promise<void>;
+    /**
+     * Extract conversation context from messages
+     */
+    private extractConversationContext;
+    /**
+     * Get current turn number
+     */
+    private getTurnNumber;
+    /**
+     * Calculate cost based on tokens and model
+     */
+    calculateCost(tokens: {
+        prompt: number;
+        completion: number;
+        total: number;
+    }, model: string): number;
+    /**
+     * Get trace ID
+     */
+    getTraceId(): string;
+    /**
+     * Check if connected
+     */
+    isConnectedToServer(): boolean;
+    /**
+     * Get current event queue size
+     */
+    getQueueSize(): number;
+    /**
+     * Clear event queue
+     */
+    clearQueue(): void;
+    /**
+     * Shutdown tracer
+     */
+    shutdown(): Promise<void>;
+}
+/**
+ * Create a new OpenAI tracer instance
+ */
+declare function createOpenAITracer(config?: OpenAITraceConfig): OpenAITracer;
+/**
+ * OpenAI Function Calling Wrapper
+ * Wraps OpenAI API calls with automatic tracing
+ */
+declare class TracedOpenAI {
+    private openai;
+    private tracer;
+    constructor(openaiClient: any, tracer: OpenAITracer);
+    /**
+     * Traced chat completion with function calling
+     */
+    createChatCompletion(params: {
+        model: string;
+        messages: OpenAIMessage[];
+        tools?: OpenAITool[];
+        tool_choice?: 'auto' | 'none' | {
+            type: 'function';
+            function: {
+                name: string;
+            };
+        };
+        temperature?: number;
+        max_tokens?: number;
+    }): Promise<any>;
+}
+
+export { type OpenAIFunctionCall, type OpenAIMessage, type OpenAITool, type OpenAIToolCall, type OpenAITraceConfig, type OpenAITraceEvent, OpenAITracer, TracedOpenAI, createOpenAITracer, OpenAITracer as default };

package/dist/index.d.ts

@@ -0,0 +1,214 @@
+import { EventEmitter } from 'events';
+
+/**
+ * OpenAI Function Calling Tracer
+ * Tracks OpenAI function calling agents with detailed event logging
+ */
+
+interface OpenAIFunctionCall {
+    id: string;
+    type: 'function';
+    function: {
+        name: string;
+        arguments: string;
+    };
+}
+interface OpenAIToolCall {
+    id: string;
+    type: 'function';
+    function: {
+        name: string;
+        arguments: string;
+    };
+}
+interface OpenAIMessage {
+    role: 'system' | 'user' | 'assistant' | 'tool';
+    content: string | null;
+    tool_calls?: OpenAIToolCall[];
+    tool_call_id?: string;
+    name?: string;
+}
+interface OpenAITool {
+    type: 'function';
+    function: {
+        name: string;
+        description: string;
+        parameters: any;
+    };
+}
+interface OpenAITraceEvent {
+    eventId: string;
+    traceId: string;
+    timestamp: number;
+    type: 'function_call_start' | 'function_call_end' | 'tool_selection' | 'conversation_turn' | 'error';
+    data: any;
+    metadata?: Record<string, any>;
+}
+interface OpenAITraceConfig {
+    projectName?: string;
+    endpoint?: string;
+    metadata?: Record<string, any>;
+    autoConnect?: boolean;
+}
+/**
+ * OpenAI Function Calling Tracer
+ *
+ * This class provides comprehensive tracing capabilities for OpenAI function calling agents.
+ * It tracks function calls, tool selections, conversation turns, and errors, sending
+ * real-time events to the Agent Trace Visualizer backend via WebSocket.
+ *
+ * Features:
+ * - Real-time event queuing and flushing
+ * - WebSocket connection management
+ * - Cost calculation for different OpenAI models
+ * - Error tracking and reporting
+ * - Automatic reconnection handling
+ */
+declare class OpenAITracer extends EventEmitter {
+    private traceId;
+    private projectName;
+    private endpoint;
+    private metadata;
+    private eventQueue;
+    private isConnected;
+    private client;
+    private flushInterval;
+    /**
+     * Creates a new OpenAI tracer instance
+     *
+     * @param config - Configuration object for the tracer
+     * @param config.projectName - Project name for organizing traces (default: 'openai-agent')
+     * @param config.endpoint - Backend server endpoint (default: 'http://localhost:3000')
+     * @param config.metadata - Additional metadata to include with all events
+     * @param config.autoConnect - Whether to automatically connect to the server (default: true)
+     */
+    constructor(config?: OpenAITraceConfig);
+    /**
+     * Establishes WebSocket connection to the trace server
+     *
+     * This method creates a Socket.IO client connection to the backend server,
+     * sets up event listeners for connection status, and starts the periodic
+     * event flushing mechanism.
+     *
+     * @throws Error if connection fails
+     */
+    connect(): Promise<void>;
+    /**
+     * Disconnect from the trace server
+     */
+    disconnect(): void;
+    /**
+     * Records the start of a function call execution
+     *
+     * This method creates and queues a function call start event, capturing
+     * the function name, arguments, model, messages, and available tools.
+     * It returns an event ID that can be used to correlate with the end event.
+     *
+     * @param functionName - Name of the function being called
+     * @param functionArguments - Arguments passed to the function
+     * @param model - OpenAI model being used (e.g., 'gpt-4', 'gpt-3.5-turbo')
+     * @param messages - Array of conversation messages
+     * @param tools - Optional array of available tools
+     * @returns Event ID for correlating with the corresponding end event
+     */
+    traceFunctionCallStart(functionName: string, functionArguments: any, model: string, messages: OpenAIMessage[], tools?: OpenAITool[]): string;
+    /**
+     * Track function call end
+     */
+    traceFunctionCallEnd(eventId: string, result: any, cost: number, latency: number, tokens?: {
+        prompt: number;
+        completion: number;
+        total: number;
+    }): void;
+    /**
+     * Track tool selection
+     */
+    traceToolSelection(availableTools: OpenAITool[], selectedTool: OpenAITool, reasoning?: string, confidence?: number): void;
+    /**
+     * Track conversation turn
+     */
+    traceConversationTurn(userMessage: string, assistantResponse: string, model: string, tokens?: {
+        prompt: number;
+        completion: number;
+        total: number;
+    }, cost?: number): void;
+    /**
+     * Track error
+     */
+    traceError(error: Error, context: string, functionName?: string, functionArguments?: any): void;
+    /**
+     * Add event to queue
+     */
+    private addEvent;
+    /**
+     * Flush events to server
+     */
+    flushQueue(): Promise<void>;
+    /**
+     * Extract conversation context from messages
+     */
+    private extractConversationContext;
+    /**
+     * Get current turn number
+     */
+    private getTurnNumber;
+    /**
+     * Calculate cost based on tokens and model
+     */
+    calculateCost(tokens: {
+        prompt: number;
+        completion: number;
+        total: number;
+    }, model: string): number;
+    /**
+     * Get trace ID
+     */
+    getTraceId(): string;
+    /**
+     * Check if connected
+     */
+    isConnectedToServer(): boolean;
+    /**
+     * Get current event queue size
+     */
+    getQueueSize(): number;
+    /**
+     * Clear event queue
+     */
+    clearQueue(): void;
+    /**
+     * Shutdown tracer
+     */
+    shutdown(): Promise<void>;
+}
+/**
+ * Create a new OpenAI tracer instance
+ */
+declare function createOpenAITracer(config?: OpenAITraceConfig): OpenAITracer;
+/**
+ * OpenAI Function Calling Wrapper
+ * Wraps OpenAI API calls with automatic tracing
+ */
+declare class TracedOpenAI {
+    private openai;
+    private tracer;
+    constructor(openaiClient: any, tracer: OpenAITracer);
+    /**
+     * Traced chat completion with function calling
+     */
+    createChatCompletion(params: {
+        model: string;
+        messages: OpenAIMessage[];
+        tools?: OpenAITool[];
+        tool_choice?: 'auto' | 'none' | {
+            type: 'function';
+            function: {
+                name: string;
+            };
+        };
+        temperature?: number;
+        max_tokens?: number;
+    }): Promise<any>;
+}
+
+export { type OpenAIFunctionCall, type OpenAIMessage, type OpenAITool, type OpenAIToolCall, type OpenAITraceConfig, type OpenAITraceEvent, OpenAITracer, TracedOpenAI, createOpenAITracer, OpenAITracer as default };