openlit 1.0.0

package/src/index.ts

@@ -0,0 +1,72 @@
+import { Resource } from '@opentelemetry/resources';
+import {
+  SEMRESATTRS_DEPLOYMENT_ENVIRONMENT,
+  SEMRESATTRS_SERVICE_NAME,
+  SEMRESATTRS_TELEMETRY_SDK_NAME,
+} from '@opentelemetry/semantic-conventions';
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { OpenlitOptions } from './types';
+import Tracing from './tracing';
+import { DEFAULT_APPLICATION_NAME, DEFAULT_ENVIRONMENT, SDK_NAME } from './constant';
+import { SpanExporter } from '@opentelemetry/sdk-trace-base';
+
+export default class Openlit {
+  static resource: Resource;
+  static options: OpenlitOptions;
+  static _sdk: NodeSDK;
+  static init(options?: OpenlitOptions) {
+    try {
+      const { environment = DEFAULT_ENVIRONMENT, applicationName = DEFAULT_APPLICATION_NAME } =
+        options || {};
+
+      const otlpEndpoint =
+        options?.otlpEndpoint || process.env.OTEL_EXPORTER_OTLP_ENDPOINT || undefined;
+      let otlpHeaders = options?.otlpHeaders;
+      if (!otlpHeaders) {
+        if (process.env.OTEL_EXPORTER_OTLP_HEADERS) {
+          otlpHeaders = process.env.OTEL_EXPORTER_OTLP_HEADERS.split(',').reduce(
+            (acc: Record<string, any>, items: string) => {
+              const keyVal: string[] = items.split('=');
+              acc[keyVal[0]] = keyVal[1];
+              return acc;
+            },
+            {}
+          );
+        } else {
+          otlpHeaders = {};
+        }
+      }
+
+      this.options = options || {};
+      this.options.otlpEndpoint = otlpEndpoint;
+      this.options.otlpHeaders = otlpHeaders;
+      this.options.disableBatch =
+        options?.disableBatch === undefined ? true : !!options.disableBatch;
+
+      this.resource = new Resource({
+        [SEMRESATTRS_SERVICE_NAME]: applicationName,
+        [SEMRESATTRS_DEPLOYMENT_ENVIRONMENT]: environment,
+        [SEMRESATTRS_TELEMETRY_SDK_NAME]: SDK_NAME,
+      });
+
+      Tracing.setup({
+        ...this.options,
+        environment,
+        applicationName,
+        otlpEndpoint,
+        otlpHeaders,
+        resource: this.resource,
+      });
+
+      this._sdk = new NodeSDK({
+        resource: this.resource,
+        traceExporter: Tracing.traceExporter as SpanExporter,
+      });
+
+      // This was causing the traceProvider initilization with multiple instances.
+      // this._sdk.start();
+    } catch (e) {
+      console.log('Connection time out', e);
+    }
+  }
+}

package/src/instrumentation/anthropic/index.ts

@@ -0,0 +1,59 @@
+import {
+  InstrumentationBase,
+  InstrumentationModuleDefinition,
+  InstrumentationNodeModuleDefinition,
+  isWrapped,
+} from '@opentelemetry/instrumentation';
+import { InstrumentationConfig } from '@opentelemetry/instrumentation';
+import { INSTRUMENTATION_PREFIX } from '../../constant';
+import Anthropic from '@anthropic-ai/sdk';
+import AnthropicWrapper from './wrapper';
+
+export interface AnthropicInstrumentationConfig extends InstrumentationConfig {}
+
+export default class OpenlitAnthropicInstrumentation extends InstrumentationBase {
+  constructor(config: AnthropicInstrumentationConfig = {}) {
+    super(`${INSTRUMENTATION_PREFIX}/instrumentation-anthropic`, '1.0.0', config);
+  }
+
+  protected init(): void | InstrumentationModuleDefinition | InstrumentationModuleDefinition[] {
+    const module = new InstrumentationNodeModuleDefinition(
+      '@anthropic-ai/sdk',
+      ['>= 0.21.0'],
+      (moduleExports) => {
+        this._patch(moduleExports);
+        return moduleExports;
+      },
+      (moduleExports) => {
+        if (moduleExports !== undefined) {
+          this._unpatch(moduleExports);
+        }
+      }
+    );
+    return [module];
+  }
+
+  public manualPatch(anthropic: any): void {
+    this._patch(anthropic);
+  }
+
+  protected _patch(moduleExports: typeof Anthropic) {
+    try {
+      if (isWrapped(moduleExports.Anthropic.Messages.prototype.create)) {
+        this._unwrap(moduleExports.Anthropic.Messages.prototype, 'create');
+      }
+
+      this._wrap(
+        moduleExports.Anthropic.Messages.prototype,
+        'create',
+        AnthropicWrapper._patchMessageCreate(this.tracer)
+      );
+    } catch (e) {
+      console.error('Error in _patch method:', e);
+    }
+  }
+
+  protected _unpatch(moduleExports: typeof Anthropic) {
+    this._unwrap(moduleExports.Anthropic.Messages.prototype, 'create');
+  }
+}

package/src/instrumentation/anthropic/wrapper.ts

@@ -0,0 +1,276 @@
+import { Span, SpanKind, SpanStatusCode, Tracer, context, trace } from '@opentelemetry/api';
+import OpenlitConfig from '../../config';
+import OpenLitHelper from '../../helpers';
+import SemanticConvention from '../../semantic-convention';
+import { SDK_NAME, TELEMETRY_SDK_NAME } from '../../constant';
+
+export default class AnthropicWrapper {
+  static setBaseSpanAttributes(
+    span: any,
+    { genAIEndpoint, model, user, cost, environment, applicationName }: any
+  ) {
+    span.setAttributes({
+      [TELEMETRY_SDK_NAME]: SDK_NAME,
+    });
+
+    span.setAttribute(TELEMETRY_SDK_NAME, SDK_NAME);
+    span.setAttribute(SemanticConvention.GEN_AI_SYSTEM, SemanticConvention.GEN_AI_SYSTEM_ANTHROPIC);
+    span.setAttribute(SemanticConvention.GEN_AI_ENDPOINT, genAIEndpoint);
+    span.setAttribute(SemanticConvention.GEN_AI_ENVIRONMENT, environment);
+    span.setAttribute(SemanticConvention.GEN_AI_APPLICATION_NAME, applicationName);
+    span.setAttribute(SemanticConvention.GEN_AI_REQUEST_MODEL, model);
+    span.setAttribute(SemanticConvention.GEN_AI_REQUEST_USER, user);
+    if (cost !== undefined) span.setAttribute(SemanticConvention.GEN_AI_USAGE_COST, cost);
+
+    span.setStatus({ code: SpanStatusCode.OK });
+  }
+
+  static _patchMessageCreate(tracer: Tracer): any {
+    const genAIEndpoint = 'anthropic.resources.messages';
+    return (originalMethod: (...args: any[]) => any) => {
+      return async function (this: any, ...args: any[]) {
+        const span = tracer.startSpan(genAIEndpoint, { kind: SpanKind.CLIENT });
+        const { stream = false } = args[0];
+        return context
+          .with(trace.setSpan(context.active(), span), async () => {
+            return originalMethod.apply(this, args);
+          })
+          .then((response) => {
+            if (!!stream) {
+              return OpenLitHelper.createStreamProxy(
+                response,
+                AnthropicWrapper._messageCreateGenerator({
+                  args,
+                  genAIEndpoint,
+                  response,
+                  span,
+                })
+              );
+            }
+
+            return AnthropicWrapper._messageCreate({ args, genAIEndpoint, response, span });
+          })
+          .catch((e: any) => {
+            OpenLitHelper.handleException(span, e);
+            span.end();
+            throw e;
+          });
+      };
+    };
+  }
+
+  static async _messageCreate({
+    args,
+    genAIEndpoint,
+    response,
+    span,
+  }: {
+    args: any[];
+    genAIEndpoint: string;
+    response: any;
+    span: Span;
+  }) {
+    try {
+      await AnthropicWrapper._messageCreateCommonSetter({
+        args,
+        genAIEndpoint,
+        result: response,
+        span,
+      });
+      return response;
+    } catch (e: any) {
+      OpenLitHelper.handleException(span, e);
+    } finally {
+      span.end();
+    }
+  }
+
+  static async *_messageCreateGenerator({
+    args,
+    genAIEndpoint,
+    response,
+    span,
+  }: {
+    args: any[];
+    genAIEndpoint: string;
+    response: any;
+    span: Span;
+  }) {
+    try {
+      const result = {
+        id: '0',
+        model: '',
+        stop_reason: '',
+        content: [
+          {
+            text: '',
+            role: '',
+          },
+        ],
+        usage: {
+          input_tokens: 0,
+          output_tokens: 0,
+          total_tokens: 0,
+        },
+      };
+      for await (const chunk of response) {
+        switch (chunk.type) {
+          case 'content_block_delta':
+            result.content[0].text += chunk.delta?.text ?? '';
+            break;
+          case 'message_stop':
+            break;
+
+          case 'content_block_stop':
+            break;
+
+          case 'message_start':
+            if (chunk.message) {
+              result.id = chunk.message.id;
+              result.model = chunk.message.model;
+              result.content[0].role = chunk.message.role;
+              result.usage.input_tokens += Number(chunk.message.usage?.input_tokens) ?? 0;
+              result.usage.output_tokens += Number(chunk.message.usage?.output_tokens) ?? 0;
+              result.stop_reason = chunk.message?.stop_reason ?? '';
+            }
+            break;
+
+          case 'content_block_start':
+            result.content[0].text = chunk.content_block?.text ?? '';
+            break;
+          case 'message_delta':
+            result.stop_reason = chunk.delta?.stop_reason ?? '';
+            result.usage.output_tokens += Number(chunk.usage?.output_tokens) ?? 0;
+            break;
+        }
+
+        yield chunk;
+      }
+
+      result.usage.total_tokens = result.usage.output_tokens + result.usage.input_tokens;
+
+      await AnthropicWrapper._messageCreateCommonSetter({
+        args,
+        genAIEndpoint,
+        result,
+        span,
+      });
+      return response;
+    } catch (e: any) {
+      OpenLitHelper.handleException(span, e);
+    } finally {
+      span.end();
+    }
+  }
+
+  static async _messageCreateCommonSetter({
+    args,
+    genAIEndpoint,
+    result,
+    span,
+  }: {
+    args: any[];
+    genAIEndpoint: string;
+    result: any;
+    span: Span;
+  }) {
+    const applicationName = OpenlitConfig.applicationName;
+    const environment = OpenlitConfig.environment;
+    const traceContent = OpenlitConfig.traceContent;
+    const {
+      messages,
+      max_tokens = null,
+      seed = null,
+      temperature = 1,
+      top_p,
+      top_k,
+      user,
+      stream = false,
+      stop_reason,
+    } = args[0];
+
+    // Format 'messages' into a single string
+    const messagePrompt = messages || '';
+    const formattedMessages = [];
+
+    for (const message of messagePrompt) {
+      const role = message.role;
+      const content = message.content;
+
+      if (Array.isArray(content)) {
+        const contentStr = content
+          .map((item) => {
+            if ('type' in item) {
+              return `${item.type}: ${item.text ? item.text : item.image_url}`;
+            } else {
+              return `text: ${item.text}`;
+            }
+          })
+          .join(', ');
+        formattedMessages.push(`${role}: ${contentStr}`);
+      } else {
+        formattedMessages.push(`${role}: ${content}`);
+      }
+    }
+
+    const prompt = formattedMessages.join('\n');
+    span.setAttribute(SemanticConvention.GEN_AI_TYPE, SemanticConvention.GEN_AI_TYPE_CHAT);
+    span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_ID, result.id);
+
+    const model = result.model || 'claude-3-sonnet-20240229';
+
+    const pricingInfo = await OpenlitConfig.updatePricingJson(OpenlitConfig.pricing_json);
+
+    // Calculate cost of the operation
+    const cost = OpenLitHelper.getChatModelCost(
+      model,
+      pricingInfo,
+      result.usage.input_tokens,
+      result.usage.output_tokens
+    );
+
+    AnthropicWrapper.setBaseSpanAttributes(span, {
+      genAIEndpoint,
+      model,
+      user,
+      cost,
+      applicationName,
+      environment,
+    });
+
+    // Request Params attributes : Start
+    span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TOP_P, top_p);
+    span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TOP_K, top_k);
+    span.setAttribute(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, max_tokens);
+    span.setAttribute(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, temperature);
+
+    span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, stop_reason);
+    span.setAttribute(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, stream);
+    span.setAttribute(SemanticConvention.GEN_AI_REQUEST_SEED, seed);
+    if (traceContent) {
+      span.setAttribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, prompt);
+    }
+    // Request Params attributes : End
+
+    span.setAttribute(SemanticConvention.GEN_AI_USAGE_PROMPT_TOKENS, result.usage.input_tokens);
+    span.setAttribute(
+      SemanticConvention.GEN_AI_USAGE_COMPLETION_TOKENS,
+      result.usage.output_tokens
+    );
+    span.setAttribute(
+      SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS,
+      result.usage.input_tokens + result.usage.output_tokens
+    );
+
+    if (result.stop_reason) {
+      span.setAttribute(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, result.stop_reason);
+    }
+
+    if (traceContent) {
+      span.setAttribute(
+        SemanticConvention.GEN_AI_CONTENT_COMPLETION,
+        result.content?.[0]?.text || ''
+      );
+    }
+  }
+}

package/src/instrumentation/index.ts

@@ -0,0 +1,66 @@
+import { registerInstrumentations } from '@opentelemetry/instrumentation';
+import { InstrumentationType, OpenlitInstrumentations } from '../types';
+
+import { TracerProvider } from '@opentelemetry/api';
+import OpenAIInstrumentation from './openai';
+import AnthropicInstrumentation from './anthropic';
+
+export default class Instrumentations {
+  static availableInstrumentations: OpenlitInstrumentations = {
+    openai: new OpenAIInstrumentation(),
+    anthropic: new AnthropicInstrumentation(),
+  };
+
+  static setup(
+    tracerProvider: TracerProvider,
+    disabledInstrumentations: string[] = [],
+    instrumentations?: OpenlitInstrumentations
+  ) {
+    if (instrumentations === undefined) {
+      const filteredInstrumentations = this.getFilteredInstrumentations(disabledInstrumentations);
+      registerInstrumentations({
+        instrumentations: filteredInstrumentations.map(([_, instrumentation]) => instrumentation),
+        tracerProvider,
+      });
+    } else {
+      const filteredInstrumentations = this.getFilteredInstrumentations(
+        disabledInstrumentations,
+        instrumentations
+      );
+      filteredInstrumentations.forEach(([k, instrumentation]) => {
+        if (this.availableInstrumentations[k].setTracerProvider) {
+          this.availableInstrumentations[k].setTracerProvider(tracerProvider);
+        }
+        if (this.availableInstrumentations[k].manualPatch) {
+          this.availableInstrumentations[k].manualPatch(instrumentation);
+        }
+      });
+      registerInstrumentations({
+        tracerProvider,
+      });
+    }
+  }
+
+  static getFilteredInstrumentations(
+    disabledInstrumentations: string[],
+    instrumentations?: OpenlitInstrumentations
+  ): [InstrumentationType, any][] {
+    const availableInstrumentations = instrumentations || this.availableInstrumentations;
+    return Object.keys(availableInstrumentations)
+      .filter((k) => {
+        if (disabledInstrumentations.includes(k)) {
+          if (typeof availableInstrumentations[k as InstrumentationType].disable === 'function') {
+            availableInstrumentations[k as InstrumentationType].disable();
+          }
+          return false;
+        }
+
+        if (typeof availableInstrumentations[k as InstrumentationType].enable === 'function') {
+          availableInstrumentations[k as InstrumentationType].enable();
+        }
+
+        return true;
+      })
+      .map((k) => [k as InstrumentationType, availableInstrumentations[k as InstrumentationType]]);
+  }
+}

package/src/instrumentation/openai/index.ts

@@ -0,0 +1,114 @@
+import {
+  InstrumentationBase,
+  InstrumentationModuleDefinition,
+  InstrumentationNodeModuleDefinition,
+  isWrapped,
+} from '@opentelemetry/instrumentation';
+import { InstrumentationConfig } from '@opentelemetry/instrumentation';
+import { INSTRUMENTATION_PREFIX } from '../../constant';
+import OpenAI from 'openai';
+import OpenAIWrapper from './wrapper';
+
+export interface OpenAIInstrumentationConfig extends InstrumentationConfig {}
+
+export default class OpenlitOpenAIInstrumentation extends InstrumentationBase {
+  constructor(config: OpenAIInstrumentationConfig = {}) {
+    super(`${INSTRUMENTATION_PREFIX}/instrumentation-openai`, '1.0.0', config);
+  }
+
+  protected init(): void | InstrumentationModuleDefinition | InstrumentationModuleDefinition[] {
+    const module = new InstrumentationNodeModuleDefinition(
+      'openai',
+      ['>=3.1.0 <5'],
+      (moduleExports) => {
+        this._patch(moduleExports);
+        return moduleExports;
+      },
+      (moduleExports) => {
+        if (moduleExports !== undefined) {
+          this._unpatch(moduleExports);
+        }
+      }
+    );
+    return [module];
+  }
+
+  public manualPatch(openai: any): void {
+    this._patch(openai);
+  }
+
+  protected _patch(moduleExports: typeof OpenAI) {
+    try {
+      if (isWrapped(moduleExports.OpenAI.Chat.Completions.prototype.create)) {
+        this._unwrap(moduleExports.OpenAI.Chat.Completions.prototype, 'create');
+      }
+
+      if (isWrapped(moduleExports.OpenAI.Embeddings.prototype.create)) {
+        this._unwrap(moduleExports.OpenAI.Embeddings.prototype, 'create');
+      }
+
+      if (isWrapped(moduleExports.OpenAI.FineTuning.Jobs.prototype.create)) {
+        this._unwrap(moduleExports.OpenAI.FineTuning.Jobs.prototype, 'create');
+      }
+
+      if (isWrapped(moduleExports.OpenAI.Images.prototype.generate)) {
+        this._unwrap(moduleExports.OpenAI.Images.prototype, 'generate');
+      }
+
+      if (isWrapped(moduleExports.OpenAI.Images.prototype.createVariation)) {
+        this._unwrap(moduleExports.OpenAI.Images.prototype, 'createVariation');
+      }
+
+      if (isWrapped(moduleExports.OpenAI.Audio.Speech.prototype)) {
+        this._unwrap(moduleExports.OpenAI.Audio.Speech.prototype, 'create');
+      }
+
+      this._wrap(
+        moduleExports.OpenAI.Chat.Completions.prototype,
+        'create',
+        OpenAIWrapper._patchChatCompletionCreate(this.tracer)
+      );
+
+      this._wrap(
+        moduleExports.OpenAI.Embeddings.prototype,
+        'create',
+        OpenAIWrapper._patchEmbedding(this.tracer)
+      );
+
+      this._wrap(
+        moduleExports.OpenAI.FineTuning.Jobs.prototype,
+        'create',
+        OpenAIWrapper._patchFineTune(this.tracer)
+      );
+
+      this._wrap(
+        moduleExports.OpenAI.Images.prototype,
+        'generate',
+        OpenAIWrapper._patchImageGenerate(this.tracer)
+      );
+
+      this._wrap(
+        moduleExports.OpenAI.Images.prototype,
+        'createVariation',
+        OpenAIWrapper._patchImageVariation(this.tracer)
+      );
+
+      this._wrap(
+        moduleExports.OpenAI.Audio.Speech.prototype,
+        'create',
+        OpenAIWrapper._patchAudioCreate(this.tracer)
+      );
+    } catch (e) {
+      console.error('Error in _patch method:', e);
+    }
+  }
+
+  protected _unpatch(moduleExports: typeof OpenAI) {
+    this._unwrap(moduleExports.OpenAI.Chat.Completions.prototype, 'create');
+    this._unwrap(moduleExports.OpenAI.Embeddings.prototype, 'create');
+    this._unwrap(moduleExports.OpenAI.FineTuning.prototype, 'jobs');
+    this._unwrap(moduleExports.OpenAI.Images.prototype, 'generate');
+    this._unwrap(moduleExports.OpenAI.Images.prototype, 'createVariation');
+    this._unwrap(moduleExports.OpenAI.Audio.prototype, 'speech');
+  }
+}