@sentienguard/apm 1.0.0 → 1.0.2

package/src/openai.js

@@ -0,0 +1,520 @@
+/**
+ * OpenAI Instrumentation Module
+ *
+ * Wraps OpenAI client to track:
+ * - API calls (chat completions, embeddings, images, etc.)
+ * - Latency per operation type
+ * - Token usage (prompt + completion)
+ * - Estimated costs
+ * - Error rates
+ *
+ * Usage:
+ *   import { instrumentOpenAI } from '@sentienguard/apm';
+ *   import OpenAI from 'openai';
+ *
+ *   const openai = new OpenAI({ apiKey: '...' });
+ *   instrumentOpenAI(openai);
+ */
+
+import config, { debug, warn } from './config.js';
+import { getAggregator } from './aggregator.js';
+
+// Pricing per 1K tokens (as of 2024, update as needed)
+const PRICING = {
+  'gpt-4o': { prompt: 0.005, completion: 0.015 },
+  'gpt-4o-mini': { prompt: 0.00015, completion: 0.0006 },
+  'gpt-4-turbo': { prompt: 0.01, completion: 0.03 },
+  'gpt-4': { prompt: 0.03, completion: 0.06 },
+  'gpt-3.5-turbo': { prompt: 0.0005, completion: 0.0015 },
+  'text-embedding-3-small': { prompt: 0.00002, completion: 0 },
+  'text-embedding-3-large': { prompt: 0.00013, completion: 0 },
+  'text-embedding-ada-002': { prompt: 0.0001, completion: 0 },
+  'dall-e-3': { prompt: 0.04, completion: 0 }, // per image, not tokens
+  'dall-e-2': { prompt: 0.02, completion: 0 },
+  'whisper-1': { prompt: 0.006, completion: 0 }, // per minute
+  'tts-1': { prompt: 0.015, completion: 0 }, // per 1K chars
+  'tts-1-hd': { prompt: 0.03, completion: 0 },
+};
+
+// Default pricing for unknown models
+const DEFAULT_PRICING = { prompt: 0.01, completion: 0.03 };
+
+let originalClient = null;
+
+/**
+ * Calculate estimated cost based on model and token usage
+ */
+function calculateCost(model, promptTokens, completionTokens) {
+  const pricing = PRICING[model] || DEFAULT_PRICING;
+  const promptCost = (promptTokens / 1000) * pricing.prompt;
+  const completionCost = (completionTokens / 1000) * pricing.completion;
+  return promptCost + completionCost;
+}
+
+/**
+ * Extract model name, normalizing variants
+ */
+function normalizeModel(model) {
+  if (!model) return 'unknown';
+
+  // Handle dated versions like gpt-4-0125-preview
+  if (model.startsWith('gpt-4o-mini')) return 'gpt-4o-mini';
+  if (model.startsWith('gpt-4o')) return 'gpt-4o';
+  if (model.startsWith('gpt-4-turbo')) return 'gpt-4-turbo';
+  if (model.startsWith('gpt-4')) return 'gpt-4';
+  if (model.startsWith('gpt-3.5-turbo')) return 'gpt-3.5-turbo';
+  if (model.startsWith('text-embedding-3-small')) return 'text-embedding-3-small';
+  if (model.startsWith('text-embedding-3-large')) return 'text-embedding-3-large';
+  if (model.startsWith('text-embedding-ada')) return 'text-embedding-ada-002';
+
+  return model;
+}
+
+/**
+ * Wrap a method to track its execution
+ */
+function wrapMethod(obj, methodName, operationType, extractMetrics) {
+  const original = obj[methodName];
+  if (typeof original !== 'function') return;
+
+  obj[methodName] = async function (...args) {
+    if (!config.openai?.enabled) {
+      return original.apply(this, args);
+    }
+
+    const startTime = process.hrtime.bigint();
+    let error = null;
+    let result = null;
+
+    try {
+      result = await original.apply(this, args);
+      return result;
+    } catch (err) {
+      error = err;
+      throw err;
+    } finally {
+      const latencyNs = process.hrtime.bigint() - startTime;
+      const latencyMs = Number(latencyNs) / 1e6;
+
+      // Extract metrics from args and result
+      const metrics = extractMetrics(args, result, error);
+
+      const aggregator = getAggregator();
+      aggregator.recordOpenAIOperation({
+        operation: operationType,
+        model: metrics.model,
+        latency: latencyMs,
+        promptTokens: metrics.promptTokens || 0,
+        completionTokens: metrics.completionTokens || 0,
+        totalTokens: metrics.totalTokens || 0,
+        cost: metrics.cost || 0,
+        error: error ? (error.message || 'Unknown error') : null,
+        statusCode: error?.status || (result ? 200 : null)
+      });
+
+      if (error) {
+        warn(`OpenAI ${operationType} failed: ${error.message}`);
+      } else {
+        debug(`OpenAI ${operationType} completed in ${latencyMs.toFixed(2)}ms (${metrics.totalTokens} tokens, $${metrics.cost.toFixed(6)})`);
+      }
+    }
+  };
+
+  debug(`Wrapped OpenAI method: ${methodName}`);
+}
+
+/**
+ * Instrument OpenAI chat completions
+ */
+function instrumentChatCompletions(client) {
+  if (!client.chat?.completions?.create) {
+    debug('OpenAI chat.completions.create not found');
+    return;
+  }
+
+  const original = client.chat.completions.create.bind(client.chat.completions);
+
+  client.chat.completions.create = async function (params, options) {
+    if (!config.openai?.enabled) {
+      return original(params, options);
+    }
+
+    const startTime = process.hrtime.bigint();
+    const model = normalizeModel(params.model);
+    let error = null;
+    let result = null;
+
+    try {
+      result = await original(params, options);
+      return result;
+    } catch (err) {
+      error = err;
+      throw err;
+    } finally {
+      const latencyNs = process.hrtime.bigint() - startTime;
+      const latencyMs = Number(latencyNs) / 1e6;
+
+      // Extract token usage
+      const usage = result?.usage || {};
+      const promptTokens = usage.prompt_tokens || 0;
+      const completionTokens = usage.completion_tokens || 0;
+      const totalTokens = usage.total_tokens || promptTokens + completionTokens;
+      const cost = calculateCost(model, promptTokens, completionTokens);
+
+      const aggregator = getAggregator();
+      aggregator.recordOpenAIOperation({
+        operation: 'chat.completions',
+        model,
+        latency: latencyMs,
+        promptTokens,
+        completionTokens,
+        totalTokens,
+        cost,
+        error: error ? (error.message || 'Unknown error') : null,
+        statusCode: error?.status || 200
+      });
+
+      if (error) {
+        warn(`OpenAI chat.completions failed: ${error.message}`);
+      } else {
+        debug(`OpenAI chat.completions [${model}] completed in ${latencyMs.toFixed(2)}ms (${totalTokens} tokens, $${cost.toFixed(6)})`);
+      }
+    }
+  };
+
+  debug('Instrumented OpenAI chat.completions.create');
+}
+
+/**
+ * Instrument OpenAI embeddings
+ */
+function instrumentEmbeddings(client) {
+  if (!client.embeddings?.create) {
+    debug('OpenAI embeddings.create not found');
+    return;
+  }
+
+  const original = client.embeddings.create.bind(client.embeddings);
+
+  client.embeddings.create = async function (params, options) {
+    if (!config.openai?.enabled) {
+      return original(params, options);
+    }
+
+    const startTime = process.hrtime.bigint();
+    const model = normalizeModel(params.model);
+    let error = null;
+    let result = null;
+
+    try {
+      result = await original(params, options);
+      return result;
+    } catch (err) {
+      error = err;
+      throw err;
+    } finally {
+      const latencyNs = process.hrtime.bigint() - startTime;
+      const latencyMs = Number(latencyNs) / 1e6;
+
+      const usage = result?.usage || {};
+      const promptTokens = usage.prompt_tokens || 0;
+      const totalTokens = usage.total_tokens || promptTokens;
+      const cost = calculateCost(model, promptTokens, 0);
+
+      const aggregator = getAggregator();
+      aggregator.recordOpenAIOperation({
+        operation: 'embeddings',
+        model,
+        latency: latencyMs,
+        promptTokens,
+        completionTokens: 0,
+        totalTokens,
+        cost,
+        error: error ? (error.message || 'Unknown error') : null,
+        statusCode: error?.status || 200
+      });
+
+      if (error) {
+        warn(`OpenAI embeddings failed: ${error.message}`);
+      } else {
+        debug(`OpenAI embeddings [${model}] completed in ${latencyMs.toFixed(2)}ms (${totalTokens} tokens, $${cost.toFixed(6)})`);
+      }
+    }
+  };
+
+  debug('Instrumented OpenAI embeddings.create');
+}
+
+/**
+ * Instrument OpenAI images (DALL-E)
+ */
+function instrumentImages(client) {
+  if (!client.images?.generate) {
+    debug('OpenAI images.generate not found');
+    return;
+  }
+
+  const original = client.images.generate.bind(client.images);
+
+  client.images.generate = async function (params, options) {
+    if (!config.openai?.enabled) {
+      return original(params, options);
+    }
+
+    const startTime = process.hrtime.bigint();
+    const model = normalizeModel(params.model || 'dall-e-2');
+    const numImages = params.n || 1;
+    let error = null;
+    let result = null;
+
+    try {
+      result = await original(params, options);
+      return result;
+    } catch (err) {
+      error = err;
+      throw err;
+    } finally {
+      const latencyNs = process.hrtime.bigint() - startTime;
+      const latencyMs = Number(latencyNs) / 1e6;
+
+      // DALL-E pricing is per image, not tokens
+      const pricing = PRICING[model] || { prompt: 0.02 };
+      const cost = pricing.prompt * numImages;
+
+      const aggregator = getAggregator();
+      aggregator.recordOpenAIOperation({
+        operation: 'images.generate',
+        model,
+        latency: latencyMs,
+        promptTokens: 0,
+        completionTokens: 0,
+        totalTokens: 0,
+        cost,
+        error: error ? (error.message || 'Unknown error') : null,
+        statusCode: error?.status || 200,
+        metadata: { numImages }
+      });
+
+      if (error) {
+        warn(`OpenAI images.generate failed: ${error.message}`);
+      } else {
+        debug(`OpenAI images.generate [${model}] completed in ${latencyMs.toFixed(2)}ms (${numImages} images, $${cost.toFixed(4)})`);
+      }
+    }
+  };
+
+  debug('Instrumented OpenAI images.generate');
+}
+
+/**
+ * Instrument OpenAI audio (Whisper, TTS)
+ */
+function instrumentAudio(client) {
+  // Transcriptions (Whisper)
+  if (client.audio?.transcriptions?.create) {
+    const originalTranscribe = client.audio.transcriptions.create.bind(client.audio.transcriptions);
+
+    client.audio.transcriptions.create = async function (params, options) {
+      if (!config.openai?.enabled) {
+        return originalTranscribe(params, options);
+      }
+
+      const startTime = process.hrtime.bigint();
+      const model = normalizeModel(params.model || 'whisper-1');
+      let error = null;
+      let result = null;
+
+      try {
+        result = await originalTranscribe(params, options);
+        return result;
+      } catch (err) {
+        error = err;
+        throw err;
+      } finally {
+        const latencyNs = process.hrtime.bigint() - startTime;
+        const latencyMs = Number(latencyNs) / 1e6;
+
+        // Whisper pricing is per minute of audio
+        // We don't know the exact duration without parsing the file
+        const cost = 0; // Would need file duration
+
+        const aggregator = getAggregator();
+        aggregator.recordOpenAIOperation({
+          operation: 'audio.transcriptions',
+          model,
+          latency: latencyMs,
+          promptTokens: 0,
+          completionTokens: 0,
+          totalTokens: 0,
+          cost,
+          error: error ? (error.message || 'Unknown error') : null,
+          statusCode: error?.status || 200
+        });
+
+        if (error) {
+          warn(`OpenAI audio.transcriptions failed: ${error.message}`);
+        } else {
+          debug(`OpenAI audio.transcriptions [${model}] completed in ${latencyMs.toFixed(2)}ms`);
+        }
+      }
+    };
+
+    debug('Instrumented OpenAI audio.transcriptions.create');
+  }
+
+  // Text-to-Speech
+  if (client.audio?.speech?.create) {
+    const originalSpeech = client.audio.speech.create.bind(client.audio.speech);
+
+    client.audio.speech.create = async function (params, options) {
+      if (!config.openai?.enabled) {
+        return originalSpeech(params, options);
+      }
+
+      const startTime = process.hrtime.bigint();
+      const model = normalizeModel(params.model || 'tts-1');
+      const inputLength = params.input?.length || 0;
+      let error = null;
+      let result = null;
+
+      try {
+        result = await originalSpeech(params, options);
+        return result;
+      } catch (err) {
+        error = err;
+        throw err;
+      } finally {
+        const latencyNs = process.hrtime.bigint() - startTime;
+        const latencyMs = Number(latencyNs) / 1e6;
+
+        // TTS pricing is per 1K characters
+        const pricing = PRICING[model] || { prompt: 0.015 };
+        const cost = (inputLength / 1000) * pricing.prompt;
+
+        const aggregator = getAggregator();
+        aggregator.recordOpenAIOperation({
+          operation: 'audio.speech',
+          model,
+          latency: latencyMs,
+          promptTokens: 0,
+          completionTokens: 0,
+          totalTokens: 0,
+          cost,
+          error: error ? (error.message || 'Unknown error') : null,
+          statusCode: error?.status || 200,
+          metadata: { inputLength }
+        });
+
+        if (error) {
+          warn(`OpenAI audio.speech failed: ${error.message}`);
+        } else {
+          debug(`OpenAI audio.speech [${model}] completed in ${latencyMs.toFixed(2)}ms (${inputLength} chars, $${cost.toFixed(6)})`);
+        }
+      }
+    };
+
+    debug('Instrumented OpenAI audio.speech.create');
+  }
+}
+
+/**
+ * Instrument OpenAI moderations
+ */
+function instrumentModerations(client) {
+  if (!client.moderations?.create) {
+    debug('OpenAI moderations.create not found');
+    return;
+  }
+
+  const original = client.moderations.create.bind(client.moderations);
+
+  client.moderations.create = async function (params, options) {
+    if (!config.openai?.enabled) {
+      return original(params, options);
+    }
+
+    const startTime = process.hrtime.bigint();
+    const model = normalizeModel(params.model || 'text-moderation-latest');
+    let error = null;
+    let result = null;
+
+    try {
+      result = await original(params, options);
+      return result;
+    } catch (err) {
+      error = err;
+      throw err;
+    } finally {
+      const latencyNs = process.hrtime.bigint() - startTime;
+      const latencyMs = Number(latencyNs) / 1e6;
+
+      // Moderations are free
+      const aggregator = getAggregator();
+      aggregator.recordOpenAIOperation({
+        operation: 'moderations',
+        model,
+        latency: latencyMs,
+        promptTokens: 0,
+        completionTokens: 0,
+        totalTokens: 0,
+        cost: 0,
+        error: error ? (error.message || 'Unknown error') : null,
+        statusCode: error?.status || 200
+      });
+
+      if (error) {
+        warn(`OpenAI moderations failed: ${error.message}`);
+      } else {
+        debug(`OpenAI moderations completed in ${latencyMs.toFixed(2)}ms`);
+      }
+    }
+  };
+
+  debug('Instrumented OpenAI moderations.create');
+}
+
+/**
+ * Instrument an OpenAI client instance
+ *
+ * @param {OpenAI} client - OpenAI client instance
+ */
+export function instrumentOpenAI(client) {
+  if (!client) {
+    warn('instrumentOpenAI called with no client');
+    return;
+  }
+
+  if (!config.openai?.enabled) {
+    debug('OpenAI instrumentation disabled via config');
+    return;
+  }
+
+  try {
+    originalClient = client;
+
+    // Instrument all API endpoints
+    instrumentChatCompletions(client);
+    instrumentEmbeddings(client);
+    instrumentImages(client);
+    instrumentAudio(client);
+    instrumentModerations(client);
+
+    debug('OpenAI instrumentation attached successfully');
+  } catch (err) {
+    warn(`Failed to instrument OpenAI: ${err.message}`);
+  }
+}
+
+/**
+ * Stop OpenAI instrumentation
+ * Note: This doesn't restore original methods (client should be discarded)
+ */
+export function stopOpenAIInstrumentation() {
+  originalClient = null;
+  debug('OpenAI instrumentation stopped');
+}
+
+export default {
+  instrumentOpenAI,
+  stopOpenAIInstrumentation
+};

package/src/transport.js

@@ -120,7 +120,7 @@ async function flush() {
   // Get and reset metrics
   const payload = aggregator.flush();
 
-  debug(`Flushing ${payload.requests.length} request metrics, ${payload.dependencies.length} dependency metrics`);
+  debug(`Flushing ${payload.requests.length} request metrics, ${payload.dependencies.length} dependency metrics, ${payload.openai?.length || 0} OpenAI metrics`);
 
   try {
     const startTime = Date.now();