dd-trace 5.66.0 → 5.67.0

package/index.d.ts

@@ -2566,6 +2566,25 @@ declare namespace tracer {
       annotate (options: llmobs.AnnotationOptions): void
       annotate (span: tracer.Span | undefined, options: llmobs.AnnotationOptions): void
 
+      /**
+       * Register a processor to be called on each LLMObs span.
+       *
+       * This can be used to modify the span before it is sent to LLMObs. For example, you can modify the input/output.
+       * You can also return `null` to omit the span entirely from being sent to LLM Observability.
+       *
+       * Otherwise, if the return value from the processor is not an instance of `LLMObservabilitySpan`, the span will be dropped.
+       *
+       * To deregister the processor, call `llmobs.deregisterProcessor()`
+       * @param processor A function that will be called for each span.
+       * @throws {Error} If a processor is already registered.
+       */
+      registerProcessor (processor: ((span: LLMObservabilitySpan) => LLMObservabilitySpan | null)): void
+
+      /**
+       * Deregister a processor.
+       */
+      deregisterProcessor (): void
+
       /**
        * Submits a custom evaluation metric for a given span ID and trace ID.
        * @param spanContext The span context of the span to submit the evaluation metric for.
@@ -2579,6 +2598,25 @@ declare namespace tracer {
       flush (): void
     }
 
+    interface LLMObservabilitySpan {
+      /**
+       * The input content associated with the span.
+       */
+      input: { content: string, role?: string }[]
+
+      /**
+       * The output content associated with the span.
+       */
+      output: { content: string, role?: string }[]
+
+      /**
+       * Get a tag from the span.
+       * @param key The key of the tag to get.
+       * @returns The value of the tag, or `undefined` if the tag does not exist.
+       */
+      getTag (key: string): string | undefined
+    }
+
     interface EvaluationOptions {
       /**
        * The name of the evaluation metric

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "5.66.0",
+  "version": "5.67.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",

package/packages/dd-trace/src/llmobs/index.js

@@ -16,6 +16,7 @@ const spanProcessCh = channel('dd-trace:span:process')
 const evalMetricAppendCh = channel('llmobs:eval-metric:append')
 const flushCh = channel('llmobs:writers:flush')
 const injectCh = channel('dd-trace:span:inject')
+const registerUserSpanProcessorCh = channel('llmobs:register-processor')
 
 const LLMObsEvalMetricsWriter = require('./writers/evaluations')
 const LLMObsTagger = require('./tagger')
@@ -56,6 +57,7 @@ function enable (config) {
 
   evalMetricAppendCh.subscribe(handleEvalMetricAppend)
   flushCh.subscribe(handleFlush)
+  registerUserSpanProcessorCh.subscribe(handleRegisterProcessor)
 
   // span processing
   spanProcessor = new LLMObsSpanProcessor(config)
@@ -86,6 +88,7 @@ function disable () {
   if (flushCh.hasSubscribers) flushCh.unsubscribe(handleFlush)
   if (spanProcessCh.hasSubscribers) spanProcessCh.unsubscribe(handleSpanProcess)
   if (injectCh.hasSubscribers) injectCh.unsubscribe(handleLLMObsParentIdInjection)
+  if (registerUserSpanProcessorCh.hasSubscribers) registerUserSpanProcessorCh.unsubscribe(handleRegisterProcessor)
 
   spanWriter?.destroy()
   evalWriter?.destroy()
@@ -126,6 +129,10 @@ function handleFlush () {
   telemetry.recordUserFlush(err)
 }
 
+function handleRegisterProcessor (userSpanProcessor) {
+  spanProcessor.setUserSpanProcessor(userSpanProcessor)
+}
+
 function handleSpanProcess (data) {
   spanProcessor.process(data)
 }

package/packages/dd-trace/src/llmobs/sdk.js

@@ -23,9 +23,16 @@ const LLMObsTagger = require('./tagger')
 const { channel } = require('dc-polyfill')
 const evalMetricAppendCh = channel('llmobs:eval-metric:append')
 const flushCh = channel('llmobs:writers:flush')
+const registerUserSpanProcessorCh = channel('llmobs:register-processor')
 const NoopLLMObs = require('./noop')
 
 class LLMObs extends NoopLLMObs {
+  /**
+   * flag representing if a user span processor has been registered
+   * @type {boolean}
+   */
+  #hasUserSpanProcessor = false
+
   constructor (tracer, llmobsModule, config) {
     super(tracer)
 
@@ -309,6 +316,27 @@ class LLMObs extends NoopLLMObs {
     }
   }
 
+  registerProcessor (processor) {
+    if (!this.enabled) return
+
+    if (this.#hasUserSpanProcessor) {
+      throw new Error(
+        '[LLMObs] Only one user span processor can be registered. ' +
+        'To register a new processor, deregister the existing processor first using `llmobs.deregisterProcessor()`.'
+      )
+    }
+
+    this.#hasUserSpanProcessor = true
+    registerUserSpanProcessorCh.publish(processor)
+  }
+
+  deregisterProcessor () {
+    if (!this.enabled) return
+
+    this.#hasUserSpanProcessor = false
+    registerUserSpanProcessorCh.publish(null)
+  }
+
   submitEvaluation (llmobsSpanContext, options = {}) {
     if (!this.enabled) return
 

package/packages/dd-trace/src/llmobs/span_processor.js

@@ -34,25 +34,55 @@ const LLMObsTagger = require('./tagger')
 const tracerVersion = require('../../../../package.json').version
 const logger = require('../log')
 
+const util = require('node:util')
+
+class LLMObservabilitySpan {
+  constructor () {
+    this.input = []
+    this.output = []
+
+    this._tags = {}
+  }
+
+  getTag (key) {
+    return this._tags[key]
+  }
+}
+
 class LLMObsSpanProcessor {
+  /** @type {import('../config')} */
+  #config
+
+  /** @type {((span: LLMObservabilitySpan) => LLMObservabilitySpan | null) | null} */
+  #userSpanProcessor
+
+  /** @type {import('./writers/spans')} */
+  #writer
+
   constructor (config) {
-    this._config = config
+    this.#config = config
+  }
+
+  setUserSpanProcessor (userSpanProcessor) {
+    this.#userSpanProcessor = userSpanProcessor
   }
 
   setWriter (writer) {
-    this._writer = writer
+    this.#writer = writer
   }
 
   // TODO: instead of relying on the tagger's weakmap registry, can we use some namespaced storage correlation?
   process ({ span }) {
-    if (!this._config.llmobs.enabled) return
+    if (!this.#config.llmobs.enabled) return
     // if the span is not in our private tagger map, it is not an llmobs span
     if (!LLMObsTagger.tagMap.has(span)) return
 
     try {
       const formattedEvent = this.format(span)
       telemetry.incrementLLMObsSpanFinishedCount(span)
-      this._writer.append(formattedEvent)
+      if (formattedEvent == null) return
+
+      this.#writer.append(formattedEvent)
     } catch (e) {
       // this should be a rare case
       // we protect against unserializable properties in the format function, and in
@@ -65,6 +95,9 @@ class LLMObsSpanProcessor {
   }
 
   format (span) {
+    const llmObsSpan = new LLMObservabilitySpan()
+    let inputType, outputType
+
     const spanTags = span.context()._tags
     const mlObsTags = LLMObsTagger.tagMap.get(span)
 
@@ -78,26 +111,29 @@ class LLMObsSpanProcessor {
       meta.model_name = mlObsTags[MODEL_NAME] || 'custom'
       meta.model_provider = (mlObsTags[MODEL_PROVIDER] || 'custom').toLowerCase()
     }
+
     if (mlObsTags[METADATA]) {
-      this._addObject(mlObsTags[METADATA], meta.metadata = {})
+      this.#addObject(mlObsTags[METADATA], meta.metadata = {})
     }
+
     if (spanKind === 'llm' && mlObsTags[INPUT_MESSAGES]) {
-      input.messages = mlObsTags[INPUT_MESSAGES]
-    }
-    if (mlObsTags[INPUT_VALUE]) {
-      input.value = mlObsTags[INPUT_VALUE]
-    }
-    if (spanKind === 'llm' && mlObsTags[OUTPUT_MESSAGES]) {
-      output.messages = mlObsTags[OUTPUT_MESSAGES]
-    }
-    if (spanKind === 'embedding' && mlObsTags[INPUT_DOCUMENTS]) {
+      llmObsSpan.input = mlObsTags[INPUT_MESSAGES]
+      inputType = 'messages'
+    } else if (spanKind === 'embedding' && mlObsTags[INPUT_DOCUMENTS]) {
       input.documents = mlObsTags[INPUT_DOCUMENTS]
+    } else if (mlObsTags[INPUT_VALUE]) {
+      llmObsSpan.input = [{ role: '', content: mlObsTags[INPUT_VALUE] }]
+      inputType = 'value'
     }
-    if (mlObsTags[OUTPUT_VALUE]) {
-      output.value = mlObsTags[OUTPUT_VALUE]
-    }
-    if (spanKind === 'retrieval' && mlObsTags[OUTPUT_DOCUMENTS]) {
+
+    if (spanKind === 'llm' && mlObsTags[OUTPUT_MESSAGES]) {
+      llmObsSpan.output = mlObsTags[OUTPUT_MESSAGES]
+      outputType = 'messages'
+    } else if (spanKind === 'retrieval' && mlObsTags[OUTPUT_DOCUMENTS]) {
       output.documents = mlObsTags[OUTPUT_DOCUMENTS]
+    } else if (mlObsTags[OUTPUT_VALUE]) {
+      llmObsSpan.output = [{ role: '', content: mlObsTags[OUTPUT_VALUE] }]
+      outputType = 'value'
     }
 
     const error = spanTags.error || spanTags[ERROR_TYPE]
@@ -107,9 +143,6 @@ class LLMObsSpanProcessor {
       meta[ERROR_STACK] = spanTags[ERROR_STACK] || error.stack
     }
 
-    if (input) meta.input = input
-    if (output) meta.output = output
-
     const metrics = mlObsTags[METRICS] || {}
 
     const mlApp = mlObsTags[ML_APP]
@@ -118,12 +151,37 @@ class LLMObsSpanProcessor {
 
     const name = mlObsTags[NAME] || span._name
 
+    const tags = this.#getTags(span, mlApp, sessionId, error)
+    llmObsSpan._tags = tags
+
+    const processedSpan = this.#runProcessor(llmObsSpan)
+    if (processedSpan === null) return null
+
+    if (processedSpan.input) {
+      if (inputType === 'messages') {
+        input.messages = processedSpan.input
+      } else if (inputType === 'value') {
+        input.value = processedSpan.input[0].content
+      }
+    }
+
+    if (processedSpan.output) {
+      if (outputType === 'messages') {
+        output.messages = processedSpan.output
+      } else if (outputType === 'value') {
+        output.value = processedSpan.output[0].content
+      }
+    }
+
+    if (input) meta.input = input
+    if (output) meta.output = output
+
     const llmObsSpanEvent = {
       trace_id: span.context().toTraceId(true),
       span_id: span.context().toSpanId(),
       parent_id: parentId,
       name,
-      tags: this._processTags(span, mlApp, sessionId, error),
+      tags: this.#objectTagsToStringArrayTags(tags),
       start_ns: Math.round(span._startTime * 1e6),
       duration: Math.round(span._duration * 1e6),
       status: error ? 'error' : 'ok',
@@ -144,7 +202,7 @@ class LLMObsSpanProcessor {
   // However, we want to protect against circular references or BigInts (unserializable)
   // This function can be reused for other fields if needed
   // Messages, Documents, and Metrics are safeguarded in `llmobs/tagger.js`
-  _addObject (obj, carrier) {
+  #addObject (obj, carrier) {
     const seenObjects = new WeakSet()
     seenObjects.add(obj) // capture root object
 
@@ -176,12 +234,12 @@ class LLMObsSpanProcessor {
     add(obj, carrier)
   }
 
-  _processTags (span, mlApp, sessionId, error) {
+  #getTags (span, mlApp, sessionId, error) {
     let tags = {
-      ...this._config.parsedDdTags,
-      version: this._config.version,
-      env: this._config.env,
-      service: this._config.service,
+      ...this.#config.parsedDdTags,
+      version: this.#config.version,
+      env: this.#config.env,
+      service: this.#config.service,
       source: 'integration',
       ml_app: mlApp,
       'ddtrace.version': tracerVersion,
@@ -191,13 +249,51 @@ class LLMObsSpanProcessor {
 
     const errType = span.context()._tags[ERROR_TYPE] || error?.name
     if (errType) tags.error_type = errType
+
     if (sessionId) tags.session_id = sessionId
+
     const integration = LLMObsTagger.tagMap.get(span)?.[INTEGRATION]
     if (integration) tags.integration = integration
+
     const existingTags = LLMObsTagger.tagMap.get(span)?.[TAGS] || {}
     if (existingTags) tags = { ...tags, ...existingTags }
+
+    return tags
+  }
+
+  #objectTagsToStringArrayTags (tags) {
     return Object.entries(tags).map(([key, value]) => `${key}:${value ?? ''}`)
   }
+
+  /**
+   * Runs the user span processor, emitting telemetry and adding some guardrails against invalid return types
+   * @param {LLMObservabilitySpan} span
+   * @returns {LLMObservabilitySpan | null}
+   */
+  #runProcessor (span) {
+    const processor = this.#userSpanProcessor
+    if (!processor) return span
+
+    let error = false
+
+    try {
+      const processedLLMObsSpan = processor(span)
+      if (processedLLMObsSpan === null) return null
+
+      if (!(processedLLMObsSpan instanceof LLMObservabilitySpan)) {
+        error = true
+        logger.warn('User span processor must return an instance of an LLMObservabilitySpan or null, dropping span.')
+        return null
+      }
+
+      return processedLLMObsSpan
+    } catch (e) {
+      logger.error(`[LLMObs] Error in LLMObs span processor (${util.inspect(processor)}): ${util.inspect(e)}`)
+      error = true
+    } finally {
+      telemetry.recordLLMObsUserProcessorCalled(error)
+    }
+  }
 }
 
 module.exports = LLMObsSpanProcessor

package/packages/dd-trace/src/llmobs/telemetry.js

@@ -154,6 +154,11 @@ function recordSubmitEvaluation (options, err, value = 1) {
   llmobsMetrics.count('evals_submitted', tags).inc(value)
 }
 
+function recordLLMObsUserProcessorCalled (error, value = 1) {
+  const tags = { error: error ? 1 : 0 }
+  llmobsMetrics.count('user_processor_called', tags).inc(value)
+}
+
 module.exports = {
   recordLLMObsEnabled,
   incrementLLMObsSpanStartCount,
@@ -164,5 +169,6 @@ module.exports = {
   recordLLMObsAnnotate,
   recordUserFlush,
   recordExportSpan,
-  recordSubmitEvaluation
+  recordSubmitEvaluation,
+  recordLLMObsUserProcessorCalled
 }