dd-trace 4.47.1 → 4.49.0

package/packages/dd-trace/src/llmobs/constants/tags.js

@@ -0,0 +1,34 @@
+'use strict'
+
+module.exports = {
+  SPAN_KINDS: ['llm', 'agent', 'workflow', 'task', 'tool', 'embedding', 'retrieval'],
+  SPAN_KIND: '_ml_obs.meta.span.kind',
+  SESSION_ID: '_ml_obs.session_id',
+  METADATA: '_ml_obs.meta.metadata',
+  METRICS: '_ml_obs.metrics',
+  ML_APP: '_ml_obs.meta.ml_app',
+  PROPAGATED_PARENT_ID_KEY: '_dd.p.llmobs_parent_id',
+  PARENT_ID_KEY: '_ml_obs.llmobs_parent_id',
+  TAGS: '_ml_obs.tags',
+  NAME: '_ml_obs.name',
+  TRACE_ID: '_ml_obs.trace_id',
+  PROPAGATED_TRACE_ID_KEY: '_dd.p.llmobs_trace_id',
+  ROOT_PARENT_ID: 'undefined',
+
+  MODEL_NAME: '_ml_obs.meta.model_name',
+  MODEL_PROVIDER: '_ml_obs.meta.model_provider',
+
+  INPUT_DOCUMENTS: '_ml_obs.meta.input.documents',
+  INPUT_MESSAGES: '_ml_obs.meta.input.messages',
+  INPUT_VALUE: '_ml_obs.meta.input.value',
+
+  OUTPUT_DOCUMENTS: '_ml_obs.meta.output.documents',
+  OUTPUT_MESSAGES: '_ml_obs.meta.output.messages',
+  OUTPUT_VALUE: '_ml_obs.meta.output.value',
+
+  INPUT_TOKENS_METRIC_KEY: 'input_tokens',
+  OUTPUT_TOKENS_METRIC_KEY: 'output_tokens',
+  TOTAL_TOKENS_METRIC_KEY: 'total_tokens',
+
+  DROPPED_IO_COLLECTION_ERROR: 'dropped_io'
+}

package/packages/dd-trace/src/llmobs/constants/text.js

@@ -0,0 +1,6 @@
+'use strict'
+
+module.exports = {
+  DROPPED_VALUE_TEXT: "[This value has been dropped because this span's size exceeds the 1MB size limit.]",
+  UNSERIALIZABLE_VALUE_TEXT: 'Unserializable value'
+}

package/packages/dd-trace/src/llmobs/constants/writers.js

@@ -0,0 +1,13 @@
+'use strict'
+
+module.exports = {
+  EVP_PROXY_AGENT_BASE_PATH: 'evp_proxy/v2',
+  EVP_PROXY_AGENT_ENDPOINT: 'evp_proxy/v2/api/v2/llmobs',
+  EVP_SUBDOMAIN_HEADER_NAME: 'X-Datadog-EVP-Subdomain',
+  EVP_SUBDOMAIN_HEADER_VALUE: 'llmobs-intake',
+  AGENTLESS_SPANS_ENDPOINT: '/api/v2/llmobs',
+  AGENTLESS_EVALULATIONS_ENDPOINT: '/api/intake/llm-obs/v1/eval-metric',
+
+  EVP_PAYLOAD_SIZE_LIMIT: 5 << 20, // 5MB (actual limit is 5.1MB)
+  EVP_EVENT_SIZE_LIMIT: (1 << 20) - 1024 // 999KB (actual limit is 1MB)
+}

package/packages/dd-trace/src/llmobs/index.js

@@ -0,0 +1,103 @@
+'use strict'
+
+const log = require('../log')
+const { PROPAGATED_PARENT_ID_KEY } = require('./constants/tags')
+const { storage } = require('./storage')
+
+const LLMObsSpanProcessor = require('./span_processor')
+
+const { channel } = require('dc-polyfill')
+const spanProcessCh = channel('dd-trace:span:process')
+const evalMetricAppendCh = channel('llmobs:eval-metric:append')
+const flushCh = channel('llmobs:writers:flush')
+const injectCh = channel('dd-trace:span:inject')
+
+const LLMObsAgentlessSpanWriter = require('./writers/spans/agentless')
+const LLMObsAgentProxySpanWriter = require('./writers/spans/agentProxy')
+const LLMObsEvalMetricsWriter = require('./writers/evaluations')
+
+/**
+ * Setting writers and processor globally when LLMObs is enabled
+ * We're setting these in this module instead of on the SDK.
+ * This is to isolate any subscribers and periodic tasks to this module,
+ * and not conditionally instantiate in the SDK, since the SDK is always instantiated
+ * if the tracer is `init`ed. But, in those cases, we don't want to start writers or subscribe
+ * to channels.
+ */
+let spanProcessor
+let spanWriter
+let evalWriter
+
+function enable (config) {
+  // create writers and eval writer append and flush channels
+  // span writer append is handled by the span processor
+  evalWriter = new LLMObsEvalMetricsWriter(config)
+  spanWriter = createSpanWriter(config)
+
+  evalMetricAppendCh.subscribe(handleEvalMetricAppend)
+  flushCh.subscribe(handleFlush)
+
+  // span processing
+  spanProcessor = new LLMObsSpanProcessor(config)
+  spanProcessor.setWriter(spanWriter)
+  spanProcessCh.subscribe(handleSpanProcess)
+
+  // distributed tracing for llmobs
+  injectCh.subscribe(handleLLMObsParentIdInjection)
+}
+
+function disable () {
+  if (evalMetricAppendCh.hasSubscribers) evalMetricAppendCh.unsubscribe(handleEvalMetricAppend)
+  if (flushCh.hasSubscribers) flushCh.unsubscribe(handleFlush)
+  if (spanProcessCh.hasSubscribers) spanProcessCh.unsubscribe(handleSpanProcess)
+  if (injectCh.hasSubscribers) injectCh.unsubscribe(handleLLMObsParentIdInjection)
+
+  spanWriter?.destroy()
+  evalWriter?.destroy()
+  spanProcessor?.setWriter(null)
+
+  spanWriter = null
+  evalWriter = null
+}
+
+// since LLMObs traces can extend between services and be the same trace,
+// we need to propogate the parent id.
+function handleLLMObsParentIdInjection ({ carrier }) {
+  const parent = storage.getStore()?.span
+  if (!parent) return
+
+  const parentId = parent?.context().toSpanId()
+
+  carrier['x-datadog-tags'] += `,${PROPAGATED_PARENT_ID_KEY}=${parentId}`
+}
+
+function createSpanWriter (config) {
+  const SpanWriter = config.llmobs.agentlessEnabled ? LLMObsAgentlessSpanWriter : LLMObsAgentProxySpanWriter
+  return new SpanWriter(config)
+}
+
+function handleFlush () {
+  try {
+    spanWriter.flush()
+    evalWriter.flush()
+  } catch (e) {
+    log.warn(`Failed to flush LLMObs spans and evaluation metrics: ${e.message}`)
+  }
+}
+
+function handleSpanProcess (data) {
+  spanProcessor.process(data)
+}
+
+function handleEvalMetricAppend (payload) {
+  try {
+    evalWriter.append(payload)
+  } catch (e) {
+    log.warn(`
+      Failed to append evaluation metric to LLM Observability writer, likely due to an unserializable property.
+      Evaluation metrics won't be sent to LLM Observability: ${e.message}
+    `)
+  }
+}
+
+module.exports = { enable, disable }

package/packages/dd-trace/src/llmobs/noop.js

@@ -0,0 +1,82 @@
+'use strict'
+
+class NoopLLMObs {
+  constructor (noopTracer) {
+    this._tracer = noopTracer
+  }
+
+  get enabled () {
+    return false
+  }
+
+  enable (options) {}
+
+  disable () {}
+
+  trace (options = {}, fn) {
+    if (typeof options === 'function') {
+      fn = options
+      options = {}
+    }
+
+    const name = options.name || options.kind || fn.name
+
+    return this._tracer.trace(name, options, fn)
+  }
+
+  wrap (options = {}, fn) {
+    if (typeof options === 'function') {
+      fn = options
+      options = {}
+    }
+
+    const name = options.name || options.kind || fn.name
+
+    return this._tracer.wrap(name, options, fn)
+  }
+
+  decorate (options = {}) {
+    const llmobs = this
+    return function (target, ctxOrPropertyKey, descriptor) {
+      if (!ctxOrPropertyKey) return target
+      if (typeof ctxOrPropertyKey === 'object') {
+        const ctx = ctxOrPropertyKey
+        if (ctx.kind !== 'method') return target
+
+        return llmobs.wrap({ name: ctx.name, ...options }, target)
+      } else {
+        const propertyKey = ctxOrPropertyKey
+        if (descriptor) {
+          if (typeof descriptor.value !== 'function') return descriptor
+
+          const original = descriptor.value
+          descriptor.value = llmobs.wrap({ name: propertyKey, ...options }, original)
+
+          return descriptor
+        } else {
+          if (typeof target[propertyKey] !== 'function') return target[propertyKey]
+
+          const original = target[propertyKey]
+          Object.defineProperty(target, propertyKey, {
+            ...Object.getOwnPropertyDescriptor(target, propertyKey),
+            value: llmobs.wrap({ name: propertyKey, ...options }, original)
+          })
+
+          return target
+        }
+      }
+    }
+  }
+
+  annotate (span, options) {}
+
+  exportSpan (span) {
+    return {}
+  }
+
+  submitEvaluation (llmobsSpanContext, options) {}
+
+  flush () {}
+}
+
+module.exports = NoopLLMObs

package/packages/dd-trace/src/llmobs/plugins/base.js

@@ -0,0 +1,65 @@
+'use strict'
+
+const log = require('../../log')
+const { storage } = require('../storage')
+
+const TracingPlugin = require('../../plugins/tracing')
+const LLMObsTagger = require('../tagger')
+
+// we make this a `Plugin` so we don't have to worry about `finish` being called
+class LLMObsPlugin extends TracingPlugin {
+  constructor (...args) {
+    super(...args)
+
+    this._tagger = new LLMObsTagger(this._tracerConfig, true)
+  }
+
+  getName () {}
+
+  setLLMObsTags (ctx) {
+    throw new Error('setLLMObsTags must be implemented by the subclass')
+  }
+
+  getLLMObsSPanRegisterOptions (ctx) {
+    throw new Error('getLLMObsSPanRegisterOptions must be implemented by the subclass')
+  }
+
+  start (ctx) {
+    const oldStore = storage.getStore()
+    const parent = oldStore?.span
+    const span = ctx.currentStore?.span
+
+    const registerOptions = this.getLLMObsSPanRegisterOptions(ctx)
+
+    this._tagger.registerLLMObsSpan(span, { parent, ...registerOptions })
+  }
+
+  asyncEnd (ctx) {
+    // even though llmobs span events won't be enqueued if llmobs is disabled
+    // we should avoid doing any computations here (these listeners aren't disabled)
+    const enabled = this._tracerConfig.llmobs.enabled
+    if (!enabled) return
+
+    const span = ctx.currentStore?.span
+    if (!span) {
+      log.debug(
+        `Tried to start an LLMObs span for ${this.constructor.name} without an active APM span.
+        Not starting LLMObs span.`
+      )
+      return
+    }
+
+    this.setLLMObsTags(ctx)
+  }
+
+  configure (config) {
+    // we do not want to enable any LLMObs plugins if it is disabled on the tracer
+    const llmobsEnabled = this._tracerConfig.llmobs.enabled
+    if (llmobsEnabled === false) {
+      config = typeof config === 'boolean' ? false : { ...config, enabled: false } // override to false
+    }
+    super.configure(config)
+  }
+}
+
+module.exports = LLMObsPlugin

package/packages/dd-trace/src/llmobs/plugins/openai.js

@@ -0,0 +1,205 @@
+'use strict'
+
+const LLMObsPlugin = require('./base')
+
+class OpenAiLLMObsPlugin extends LLMObsPlugin {
+  static get prefix () {
+    return 'tracing:apm:openai:request'
+  }
+
+  getLLMObsSPanRegisterOptions (ctx) {
+    const resource = ctx.methodName
+    const methodName = gateResource(normalizeOpenAIResourceName(resource))
+    if (!methodName) return // we will not trace all openai methods for llmobs
+
+    const inputs = ctx.args[0] // completion, chat completion, and embeddings take one argument
+    const operation = getOperation(methodName)
+    const kind = operation === 'embedding' ? 'embedding' : 'llm'
+    const name = `openai.${methodName}`
+
+    return {
+      modelProvider: 'openai',
+      modelName: inputs.model,
+      kind,
+      name
+    }
+  }
+
+  setLLMObsTags (ctx) {
+    const span = ctx.currentStore?.span
+    const resource = ctx.methodName
+    const methodName = gateResource(normalizeOpenAIResourceName(resource))
+    if (!methodName) return // we will not trace all openai methods for llmobs
+
+    const inputs = ctx.args[0] // completion, chat completion, and embeddings take one argument
+    const response = ctx.result?.data // no result if error
+    const error = !!span.context()._tags.error
+
+    const operation = getOperation(methodName)
+
+    if (operation === 'completion') {
+      this._tagCompletion(span, inputs, response, error)
+    } else if (operation === 'chat') {
+      this._tagChatCompletion(span, inputs, response, error)
+    } else if (operation === 'embedding') {
+      this._tagEmbedding(span, inputs, response, error)
+    }
+
+    if (!error) {
+      const metrics = this._extractMetrics(response)
+      this._tagger.tagMetrics(span, metrics)
+    }
+  }
+
+  _extractMetrics (response) {
+    const metrics = {}
+    const tokenUsage = response.usage
+
+    if (tokenUsage) {
+      const inputTokens = tokenUsage.prompt_tokens
+      if (inputTokens) metrics.inputTokens = inputTokens
+
+      const outputTokens = tokenUsage.completion_tokens
+      if (outputTokens) metrics.outputTokens = outputTokens
+
+      const totalTokens = tokenUsage.total_toksn || (inputTokens + outputTokens)
+      if (totalTokens) metrics.totalTokens = totalTokens
+    }
+
+    return metrics
+  }
+
+  _tagEmbedding (span, inputs, response, error) {
+    const { model, ...parameters } = inputs
+
+    const metadata = {
+      encoding_format: parameters.encoding_format || 'float'
+    }
+    if (inputs.dimensions) metadata.dimensions = inputs.dimensions
+    this._tagger.tagMetadata(span, metadata)
+
+    let embeddingInputs = inputs.input
+    if (!Array.isArray(embeddingInputs)) embeddingInputs = [embeddingInputs]
+    const embeddingInput = embeddingInputs.map(input => ({ text: input }))
+
+    if (error) {
+      this._tagger.tagEmbeddingIO(span, embeddingInput, undefined)
+      return
+    }
+
+    const float = Array.isArray(response.data[0].embedding)
+    let embeddingOutput
+    if (float) {
+      const embeddingDim = response.data[0].embedding.length
+      embeddingOutput = `[${response.data.length} embedding(s) returned with size ${embeddingDim}]`
+    } else {
+      embeddingOutput = `[${response.data.length} embedding(s) returned]`
+    }
+
+    this._tagger.tagEmbeddingIO(span, embeddingInput, embeddingOutput)
+  }
+
+  _tagCompletion (span, inputs, response, error) {
+    let { prompt, model, ...parameters } = inputs
+    if (!Array.isArray(prompt)) prompt = [prompt]
+
+    const completionInput = prompt.map(p => ({ content: p }))
+
+    const completionOutput = error ? [{ content: '' }] : response.choices.map(choice => ({ content: choice.text }))
+
+    this._tagger.tagLLMIO(span, completionInput, completionOutput)
+    this._tagger.tagMetadata(span, parameters)
+  }
+
+  _tagChatCompletion (span, inputs, response, error) {
+    const { messages, model, ...parameters } = inputs
+
+    if (error) {
+      this._tagger.tagLLMIO(span, messages, [{ content: '' }])
+      return
+    }
+
+    const outputMessages = []
+    const { choices } = response
+    for (const choice of choices) {
+      const message = choice.message || choice.delta
+      const content = message.content || ''
+      const role = message.role
+
+      if (message.function_call) {
+        const functionCallInfo = {
+          name: message.function_call.name,
+          arguments: JSON.parse(message.function_call.arguments)
+        }
+        outputMessages.push({ content, role, toolCalls: [functionCallInfo] })
+      } else if (message.tool_calls) {
+        const toolCallsInfo = []
+        for (const toolCall of message.tool_calls) {
+          const toolCallInfo = {
+            arguments: JSON.parse(toolCall.function.arguments),
+            name: toolCall.function.name,
+            toolId: toolCall.id,
+            type: toolCall.type
+          }
+          toolCallsInfo.push(toolCallInfo)
+        }
+        outputMessages.push({ content, role, toolCalls: toolCallsInfo })
+      } else {
+        outputMessages.push({ content, role })
+      }
+    }
+
+    this._tagger.tagLLMIO(span, messages, outputMessages)
+
+    const metadata = Object.entries(parameters).reduce((obj, [key, value]) => {
+      if (!['tools', 'functions'].includes(key)) {
+        obj[key] = value
+      }
+
+      return obj
+    }, {})
+
+    this._tagger.tagMetadata(span, metadata)
+  }
+}
+
+// TODO: this will be moved to the APM integration
+function normalizeOpenAIResourceName (resource) {
+  switch (resource) {
+    // completions
+    case 'completions.create':
+      return 'createCompletion'
+
+    // chat completions
+    case 'chat.completions.create':
+      return 'createChatCompletion'
+
+    // embeddings
+    case 'embeddings.create':
+      return 'createEmbedding'
+    default:
+      return resource
+  }
+}
+
+function gateResource (resource) {
+  return ['createCompletion', 'createChatCompletion', 'createEmbedding'].includes(resource)
+    ? resource
+    : undefined
+}
+
+function getOperation (resource) {
+  switch (resource) {
+    case 'createCompletion':
+      return 'completion'
+    case 'createChatCompletion':
+      return 'chat'
+    case 'createEmbedding':
+      return 'embedding'
+    default:
+      // should never happen
+      return 'unknown'
+  }
+}
+
+module.exports = OpenAiLLMObsPlugin