dd-trace 5.66.0 → 5.68.0

package/packages/dd-trace/src/llmobs/sdk.js

@@ -23,9 +23,16 @@ const LLMObsTagger = require('./tagger')
 const { channel } = require('dc-polyfill')
 const evalMetricAppendCh = channel('llmobs:eval-metric:append')
 const flushCh = channel('llmobs:writers:flush')
+const registerUserSpanProcessorCh = channel('llmobs:register-processor')
 const NoopLLMObs = require('./noop')
 
 class LLMObs extends NoopLLMObs {
+  /**
+   * flag representing if a user span processor has been registered
+   * @type {boolean}
+   */
+  #hasUserSpanProcessor = false
+
   constructor (tracer, llmobsModule, config) {
     super(tracer)
 
@@ -309,6 +316,27 @@ class LLMObs extends NoopLLMObs {
     }
   }
 
+  registerProcessor (processor) {
+    if (!this.enabled) return
+
+    if (this.#hasUserSpanProcessor) {
+      throw new Error(
+        '[LLMObs] Only one user span processor can be registered. ' +
+        'To register a new processor, deregister the existing processor first using `llmobs.deregisterProcessor()`.'
+      )
+    }
+
+    this.#hasUserSpanProcessor = true
+    registerUserSpanProcessorCh.publish(processor)
+  }
+
+  deregisterProcessor () {
+    if (!this.enabled) return
+
+    this.#hasUserSpanProcessor = false
+    registerUserSpanProcessorCh.publish(null)
+  }
+
   submitEvaluation (llmobsSpanContext, options = {}) {
     if (!this.enabled) return
 

package/packages/dd-trace/src/llmobs/span_processor.js

@@ -34,25 +34,55 @@ const LLMObsTagger = require('./tagger')
 const tracerVersion = require('../../../../package.json').version
 const logger = require('../log')
 
+const util = require('node:util')
+
+class LLMObservabilitySpan {
+  constructor () {
+    this.input = []
+    this.output = []
+
+    this._tags = {}
+  }
+
+  getTag (key) {
+    return this._tags[key]
+  }
+}
+
 class LLMObsSpanProcessor {
+  /** @type {import('../config')} */
+  #config
+
+  /** @type {((span: LLMObservabilitySpan) => LLMObservabilitySpan | null) | null} */
+  #userSpanProcessor
+
+  /** @type {import('./writers/spans')} */
+  #writer
+
   constructor (config) {
-    this._config = config
+    this.#config = config
+  }
+
+  setUserSpanProcessor (userSpanProcessor) {
+    this.#userSpanProcessor = userSpanProcessor
   }
 
   setWriter (writer) {
-    this._writer = writer
+    this.#writer = writer
   }
 
   // TODO: instead of relying on the tagger's weakmap registry, can we use some namespaced storage correlation?
   process ({ span }) {
-    if (!this._config.llmobs.enabled) return
+    if (!this.#config.llmobs.enabled) return
     // if the span is not in our private tagger map, it is not an llmobs span
     if (!LLMObsTagger.tagMap.has(span)) return
 
     try {
       const formattedEvent = this.format(span)
       telemetry.incrementLLMObsSpanFinishedCount(span)
-      this._writer.append(formattedEvent)
+      if (formattedEvent == null) return
+
+      this.#writer.append(formattedEvent)
     } catch (e) {
       // this should be a rare case
       // we protect against unserializable properties in the format function, and in
@@ -65,6 +95,9 @@ class LLMObsSpanProcessor {
   }
 
   format (span) {
+    const llmObsSpan = new LLMObservabilitySpan()
+    let inputType, outputType
+
     const spanTags = span.context()._tags
     const mlObsTags = LLMObsTagger.tagMap.get(span)
 
@@ -78,26 +111,29 @@ class LLMObsSpanProcessor {
       meta.model_name = mlObsTags[MODEL_NAME] || 'custom'
       meta.model_provider = (mlObsTags[MODEL_PROVIDER] || 'custom').toLowerCase()
     }
+
     if (mlObsTags[METADATA]) {
-      this._addObject(mlObsTags[METADATA], meta.metadata = {})
+      this.#addObject(mlObsTags[METADATA], meta.metadata = {})
     }
+
     if (spanKind === 'llm' && mlObsTags[INPUT_MESSAGES]) {
-      input.messages = mlObsTags[INPUT_MESSAGES]
-    }
-    if (mlObsTags[INPUT_VALUE]) {
-      input.value = mlObsTags[INPUT_VALUE]
-    }
-    if (spanKind === 'llm' && mlObsTags[OUTPUT_MESSAGES]) {
-      output.messages = mlObsTags[OUTPUT_MESSAGES]
-    }
-    if (spanKind === 'embedding' && mlObsTags[INPUT_DOCUMENTS]) {
+      llmObsSpan.input = mlObsTags[INPUT_MESSAGES]
+      inputType = 'messages'
+    } else if (spanKind === 'embedding' && mlObsTags[INPUT_DOCUMENTS]) {
       input.documents = mlObsTags[INPUT_DOCUMENTS]
+    } else if (mlObsTags[INPUT_VALUE]) {
+      llmObsSpan.input = [{ role: '', content: mlObsTags[INPUT_VALUE] }]
+      inputType = 'value'
     }
-    if (mlObsTags[OUTPUT_VALUE]) {
-      output.value = mlObsTags[OUTPUT_VALUE]
-    }
-    if (spanKind === 'retrieval' && mlObsTags[OUTPUT_DOCUMENTS]) {
+
+    if (spanKind === 'llm' && mlObsTags[OUTPUT_MESSAGES]) {
+      llmObsSpan.output = mlObsTags[OUTPUT_MESSAGES]
+      outputType = 'messages'
+    } else if (spanKind === 'retrieval' && mlObsTags[OUTPUT_DOCUMENTS]) {
       output.documents = mlObsTags[OUTPUT_DOCUMENTS]
+    } else if (mlObsTags[OUTPUT_VALUE]) {
+      llmObsSpan.output = [{ role: '', content: mlObsTags[OUTPUT_VALUE] }]
+      outputType = 'value'
     }
 
     const error = spanTags.error || spanTags[ERROR_TYPE]
@@ -107,9 +143,6 @@ class LLMObsSpanProcessor {
       meta[ERROR_STACK] = spanTags[ERROR_STACK] || error.stack
     }
 
-    if (input) meta.input = input
-    if (output) meta.output = output
-
     const metrics = mlObsTags[METRICS] || {}
 
     const mlApp = mlObsTags[ML_APP]
@@ -118,12 +151,37 @@ class LLMObsSpanProcessor {
 
     const name = mlObsTags[NAME] || span._name
 
+    const tags = this.#getTags(span, mlApp, sessionId, error)
+    llmObsSpan._tags = tags
+
+    const processedSpan = this.#runProcessor(llmObsSpan)
+    if (processedSpan === null) return null
+
+    if (processedSpan.input) {
+      if (inputType === 'messages') {
+        input.messages = processedSpan.input
+      } else if (inputType === 'value') {
+        input.value = processedSpan.input[0].content
+      }
+    }
+
+    if (processedSpan.output) {
+      if (outputType === 'messages') {
+        output.messages = processedSpan.output
+      } else if (outputType === 'value') {
+        output.value = processedSpan.output[0].content
+      }
+    }
+
+    if (input) meta.input = input
+    if (output) meta.output = output
+
     const llmObsSpanEvent = {
       trace_id: span.context().toTraceId(true),
       span_id: span.context().toSpanId(),
       parent_id: parentId,
       name,
-      tags: this._processTags(span, mlApp, sessionId, error),
+      tags: this.#objectTagsToStringArrayTags(tags),
       start_ns: Math.round(span._startTime * 1e6),
       duration: Math.round(span._duration * 1e6),
       status: error ? 'error' : 'ok',
@@ -144,7 +202,7 @@ class LLMObsSpanProcessor {
   // However, we want to protect against circular references or BigInts (unserializable)
   // This function can be reused for other fields if needed
   // Messages, Documents, and Metrics are safeguarded in `llmobs/tagger.js`
-  _addObject (obj, carrier) {
+  #addObject (obj, carrier) {
     const seenObjects = new WeakSet()
     seenObjects.add(obj) // capture root object
 
@@ -176,12 +234,12 @@ class LLMObsSpanProcessor {
     add(obj, carrier)
   }
 
-  _processTags (span, mlApp, sessionId, error) {
+  #getTags (span, mlApp, sessionId, error) {
     let tags = {
-      ...this._config.parsedDdTags,
-      version: this._config.version,
-      env: this._config.env,
-      service: this._config.service,
+      ...this.#config.parsedDdTags,
+      version: this.#config.version,
+      env: this.#config.env,
+      service: this.#config.service,
       source: 'integration',
       ml_app: mlApp,
       'ddtrace.version': tracerVersion,
@@ -191,13 +249,51 @@ class LLMObsSpanProcessor {
 
     const errType = span.context()._tags[ERROR_TYPE] || error?.name
     if (errType) tags.error_type = errType
+
     if (sessionId) tags.session_id = sessionId
+
     const integration = LLMObsTagger.tagMap.get(span)?.[INTEGRATION]
     if (integration) tags.integration = integration
+
     const existingTags = LLMObsTagger.tagMap.get(span)?.[TAGS] || {}
     if (existingTags) tags = { ...tags, ...existingTags }
+
+    return tags
+  }
+
+  #objectTagsToStringArrayTags (tags) {
     return Object.entries(tags).map(([key, value]) => `${key}:${value ?? ''}`)
   }
+
+  /**
+   * Runs the user span processor, emitting telemetry and adding some guardrails against invalid return types
+   * @param {LLMObservabilitySpan} span
+   * @returns {LLMObservabilitySpan | null}
+   */
+  #runProcessor (span) {
+    const processor = this.#userSpanProcessor
+    if (!processor) return span
+
+    let error = false
+
+    try {
+      const processedLLMObsSpan = processor(span)
+      if (processedLLMObsSpan === null) return null
+
+      if (!(processedLLMObsSpan instanceof LLMObservabilitySpan)) {
+        error = true
+        logger.warn('User span processor must return an instance of an LLMObservabilitySpan or null, dropping span.')
+        return null
+      }
+
+      return processedLLMObsSpan
+    } catch (e) {
+      logger.error(`[LLMObs] Error in LLMObs span processor (${util.inspect(processor)}): ${util.inspect(e)}`)
+      error = true
+    } finally {
+      telemetry.recordLLMObsUserProcessorCalled(error)
+    }
+  }
 }
 
 module.exports = LLMObsSpanProcessor

package/packages/dd-trace/src/llmobs/tagger.js

@@ -20,6 +20,8 @@ const {
   NAME,
   PROPAGATED_PARENT_ID_KEY,
   ROOT_PARENT_ID,
+  CACHE_READ_INPUT_TOKENS_METRIC_KEY,
+  CACHE_WRITE_INPUT_TOKENS_METRIC_KEY,
   INPUT_TOKENS_METRIC_KEY,
   OUTPUT_TOKENS_METRIC_KEY,
   TOTAL_TOKENS_METRIC_KEY,
@@ -144,6 +146,12 @@ class LLMObsTagger {
         case 'totalTokens':
           processedKey = TOTAL_TOKENS_METRIC_KEY
           break
+        case 'cacheReadTokens':
+          processedKey = CACHE_READ_INPUT_TOKENS_METRIC_KEY
+          break
+        case 'cacheWriteTokens':
+          processedKey = CACHE_WRITE_INPUT_TOKENS_METRIC_KEY
+          break
       }
 
       if (typeof value === 'number') {

package/packages/dd-trace/src/llmobs/telemetry.js

@@ -85,7 +85,8 @@ function recordLLMObsEnabled (startTime, config, value = 1) {
     error: 0,
     agentless: Number(config.llmobs.agentlessEnabled),
     site: config.site,
-    auto: Number(autoEnabled)
+    auto: Number(autoEnabled),
+    ml_app: config.llmobs.mlApp
   }
   llmobsMetrics.count('product_enabled', tags).inc(value)
   llmobsMetrics.distribution('init_time', tags).track(initTimeMs)
@@ -154,6 +155,11 @@ function recordSubmitEvaluation (options, err, value = 1) {
   llmobsMetrics.count('evals_submitted', tags).inc(value)
 }
 
+function recordLLMObsUserProcessorCalled (error, value = 1) {
+  const tags = { error: error ? 1 : 0 }
+  llmobsMetrics.count('user_processor_called', tags).inc(value)
+}
+
 module.exports = {
   recordLLMObsEnabled,
   incrementLLMObsSpanStartCount,
@@ -164,5 +170,6 @@ module.exports = {
   recordLLMObsAnnotate,
   recordUserFlush,
   recordExportSpan,
-  recordSubmitEvaluation
+  recordSubmitEvaluation,
+  recordLLMObsUserProcessorCalled
 }

package/packages/dd-trace/src/log/index.js

@@ -1,11 +1,9 @@
 'use strict'
-
-const coalesce = require('koalas')
 const { inspect } = require('util')
 const { isTrue } = require('../util')
 const { traceChannel, debugChannel, infoChannel, warnChannel, errorChannel } = require('./channels')
 const logWriter = require('./writer')
-const { Log } = require('./log')
+const { Log, LogConfig, NoTransmitError } = require('./log')
 const { memoize } = require('./utils')
 const { getEnvironmentVariable } = require('../config-helper')
 
@@ -15,7 +13,14 @@ const config = {
   logLevel: 'debug'
 }
 
+// in most places where we know we want to mute a log we use log.error() directly
+const NO_TRANSMIT = new LogConfig(false)
+
 const log = {
+  LogConfig,
+  NO_TRANSMIT,
+  NoTransmitError,
+
   /**
    * @returns Read-only version of logging config. To modify config, call `log.use` and `log.toggle`
    */
@@ -92,18 +97,26 @@ const log = {
     return this
   },
 
+  errorWithoutTelemetry (...args) {
+    args.push(NO_TRANSMIT)
+    if (errorChannel.hasSubscribers) {
+      errorChannel.publish(Log.parse(...args))
+    }
+    return this
+  },
+
   deprecate (code, message) {
     return this._deprecate(code, message)
   },
 
   isEnabled (fleetStableConfigValue, localStableConfigValue) {
-    return isTrue(coalesce(
-      fleetStableConfigValue,
-      getEnvironmentVariable('DD_TRACE_DEBUG'),
-      getEnvironmentVariable('OTEL_LOG_LEVEL') === 'debug' || undefined,
-      localStableConfigValue,
-      config.enabled
-    ))
+    return isTrue(
+      (fleetStableConfigValue ??
+      getEnvironmentVariable('DD_TRACE_DEBUG') ??
+      getEnvironmentVariable('OTEL_LOG_LEVEL') === 'debug') ||
+      (localStableConfigValue ??
+      config.enabled)
+    )
   },
 
   getLogLevel (
@@ -111,14 +124,12 @@ const log = {
     fleetStableConfigValue,
     localStableConfigValue
   ) {
-    return coalesce(
-      optionsValue,
-      fleetStableConfigValue,
-      getEnvironmentVariable('DD_TRACE_LOG_LEVEL'),
-      getEnvironmentVariable('OTEL_LOG_LEVEL'),
-      localStableConfigValue,
+    return optionsValue ??
+      fleetStableConfigValue ??
+      getEnvironmentVariable('DD_TRACE_LOG_LEVEL') ??
+      getEnvironmentVariable('OTEL_LOG_LEVEL') ??
+      localStableConfigValue ??
       config.logLevel
-    )
   }
 }
 

package/packages/dd-trace/src/log/log.js

@@ -2,12 +2,16 @@
 
 const { format } = require('util')
 
+// other times we produce an Error in a central location and log it several other places
+class NoTransmitError extends Error {}
+
 class Log {
-  constructor (message, args, cause, delegate) {
+  constructor (message, args, cause, delegate, sendViaTelemetry = true) {
     this.message = message
     this.args = args
     this.cause = cause
     this.delegate = delegate
+    this.sendViaTelemetry = sendViaTelemetry
   }
 
   get formatted () {
@@ -22,10 +26,18 @@ class Log {
 
   static parse (...args) {
     let message, cause, delegate
+    let sendViaTelemetry = true
+
+    const maybeLogConfig = args.at(-1)
+    if (maybeLogConfig instanceof LogConfig) {
+      args.pop()
+      sendViaTelemetry = maybeLogConfig.transmit
+    }
 
-    const lastArg = args.at(-1)
-    if (lastArg && typeof lastArg === 'object' && lastArg.stack) { // lastArg instanceof Error?
+    const maybeError = args.at(-1)
+    if (maybeError && typeof maybeError === 'object' && maybeError.stack) { // maybeError instanceof Error?
       cause = args.pop()
+      if (cause instanceof NoTransmitError) sendViaTelemetry = false
     }
 
     const firstArg = args.shift()
@@ -43,10 +55,22 @@ class Log {
       message = String(firstArg)
     }
 
-    return new Log(message, args, cause, delegate)
+    return new Log(message, args, cause, delegate, sendViaTelemetry)
+  }
+}
+
+/**
+ * Pass instances of this class to logger methods when fine-grain control is needed
+ * @property {boolean} transmit - Whether to send the log via telemetry.
+ */
+class LogConfig {
+  constructor (transmit = true) {
+    this.transmit = transmit
   }
 }
 
 module.exports = {
-  Log
+  Log,
+  LogConfig,
+  NoTransmitError,
 }

package/packages/dd-trace/src/log/writer.js

@@ -75,12 +75,12 @@ function onError (err) {
   // TODO: replace it with Error(message, { cause }) when cause has broad support
   if (formatted) {
     withNoop(() => {
-      const l = Error.stackTraceLimit
+      const stackTraceLimitBackup = Error.stackTraceLimit
       Error.stackTraceLimit = 0
-      const e = new Error(formatted)
-      Error.stackTraceLimit = l
-      Error.captureStackTrace(e, stackTraceLimitFunction)
-      logger.error(e)
+      const newError = new Error(formatted)
+      Error.stackTraceLimit = stackTraceLimitBackup
+      Error.captureStackTrace(newError, stackTraceLimitFunction)
+      logger.error(newError)
     })
   }
   if (cause) withNoop(() => logger.error(cause))

package/packages/dd-trace/src/noop/span.js

@@ -22,6 +22,7 @@ class NoopSpan {
   setTag (key, value) { return this }
   addTags (keyValueMap) { return this }
   addLink (link) { return this }
+  addLinks (links) { return this }
   addSpanPointer (ptrKind, ptrDir, ptrHash) { return this }
   log () { return this }
   logEvent () {}

package/packages/dd-trace/src/opentelemetry/span.js

@@ -211,10 +211,21 @@ class Span {
     return this
   }
 
-  addLink (context, attributes) {
-    // extract dd context
+  addLink (link, attrs) {
+    // TODO: Remove this once we remove addLink(context, attrs) in v6.0.0
+    if (link instanceof SpanContext) {
+      link = { context: link, attributes: attrs ?? {} }
+    }
+
+    const { context, attributes } = link
+    // Extract dd context
     const ddSpanContext = context._ddContext
-    this._ddSpan.addLink(ddSpanContext, attributes)
+    this._ddSpan.addLink({ context: ddSpanContext, attributes })
+    return this
+  }
+
+  addLinks (links) {
+    links.forEach(link => this.addLink(link))
     return this
   }
 

package/packages/dd-trace/src/opentracing/span.js

@@ -86,8 +86,10 @@ class DatadogSpan {
 
     this._startTime = fields.startTime || this._getTime()
 
-    this._links = []
-    fields.links && fields.links.forEach(link => this.addLink(link.context, link.attributes))
+    this._links = fields.links?.map(link => ({
+      context: link.context._ddContext ?? link.context,
+      attributes: this._sanitizeAttributes(link.attributes)
+    })) ?? []
 
     if (DD_TRACE_EXPERIMENTAL_SPAN_COUNTS && finishedRegistry) {
       runtimeMetrics.increment('runtime.node.spans.unfinished')
@@ -196,13 +198,25 @@ class DatadogSpan {
 
   logEvent () {}
 
-  addLink (context, attributes) {
+  addLink (link, attrs) {
+    // TODO: Remove this once we remove addLink(context, attrs) in v6.0.0
+    if (link instanceof SpanContext) {
+      link = { context: link, attributes: attrs ?? {} }
+    }
+
+    const { context, attributes } = link
+
     this._links.push({
       context: context._ddContext ?? context,
       attributes: this._sanitizeAttributes(attributes)
     })
   }
 
+  addLinks (links) {
+    links.forEach(link => this.addLink(link))
+    return this
+  }
+
   addSpanPointer (ptrKind, ptrDir, ptrHash) {
     const zeroContext = new SpanContext({
       traceId: id('0'),
@@ -214,7 +228,7 @@ class DatadogSpan {
       'ptr.hash': ptrHash,
       'link.kind': 'span-pointer'
     }
-    this.addLink(zeroContext, attributes)
+    this.addLink({ context: zeroContext, attributes })
   }
 
   addEvent (name, attributesOrStartTime, startTime) {

package/packages/dd-trace/src/plugin_manager.js

@@ -6,6 +6,15 @@ const plugins = require('./plugins')
 const log = require('./log')
 const { getEnvironmentVariable } = require('../../dd-trace/src/config-helper')
 
+// Test optimization plugins that should only be enabled when isCiVisibility is true
+const TEST_OPTIMIZATION_PLUGINS = new Set([
+  'jest',
+  'vitest',
+  'cucumber',
+  'mocha',
+  'playwright'
+])
+
 const loadChannel = channel('dd-trace:instrumentation:load')
 
 // instrument everything that needs Plugin System V2 instrumentation
@@ -74,6 +83,13 @@ module.exports = class PluginManager {
 
     if (!Plugin) return
     if (!this._tracerConfig) return // TODO: don't wait for tracer to be initialized
+
+    // Check if this is a Test Optimization plugin and Test Optimization is not enabled
+    if (TEST_OPTIMIZATION_PLUGINS.has(name) && !this._tracerConfig.isCiVisibility) {
+      log.debug('Plugin "%s" is not initialized because Test Optimization mode is not enabled.', name)
+      return
+    }
+
     if (!this._pluginsByName[name]) {
       this._pluginsByName[name] = new Plugin(this._tracer, this._tracerConfig)
     }
@@ -148,7 +164,8 @@ module.exports = class PluginManager {
       middlewareTracingEnabled,
       traceWebsocketMessagesEnabled,
       traceWebsocketMessagesInheritSampling,
-      traceWebsocketMessagesSeparateTraces
+      traceWebsocketMessagesSeparateTraces,
+      experimental
     } = this._tracerConfig
 
     const sharedConfig = {
@@ -166,7 +183,8 @@ module.exports = class PluginManager {
       isServiceUserProvided,
       traceWebsocketMessagesEnabled,
       traceWebsocketMessagesInheritSampling,
-      traceWebsocketMessagesSeparateTraces
+      traceWebsocketMessagesSeparateTraces,
+      experimental
     }
 
     if (logInjection !== undefined) {