@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/llmobs/constants/tags.js

@@ -0,0 +1,55 @@
+'use strict'
+
+module.exports = {
+  SPAN_KINDS: ['llm', 'agent', 'workflow', 'task', 'tool', 'embedding', 'retrieval'],
+  SPAN_KIND: '_ml_obs.meta.span.kind',
+  SESSION_ID: '_ml_obs.session_id',
+  DECORATOR: '_ml_obs.decorator',
+  INTEGRATION: '_ml_obs.integration',
+  METADATA: '_ml_obs.meta.metadata',
+  METRICS: '_ml_obs.metrics',
+  ML_APP: '_ml_obs.meta.ml_app',
+  PROPAGATED_PARENT_ID_KEY: '_dd.p.llmobs_parent_id',
+  PROPAGATED_ML_APP_KEY: '_dd.p.llmobs_ml_app',
+  PARENT_ID_KEY: '_ml_obs.llmobs_parent_id',
+  TAGS: '_ml_obs.tags',
+  NAME: '_ml_obs.name',
+  TRACE_ID: '_ml_obs.trace_id',
+  PROPAGATED_TRACE_ID_KEY: '_dd.p.llmobs_trace_id',
+  ROOT_PARENT_ID: 'undefined',
+  DEFAULT_PROMPT_NAME: 'unnamed-prompt',
+  INTERNAL_CONTEXT_VARIABLE_KEYS: '_dd_context_variable_keys',
+  INTERNAL_QUERY_VARIABLE_KEYS: '_dd_query_variable_keys',
+
+  MODEL_NAME: '_ml_obs.meta.model_name',
+  MODEL_PROVIDER: '_ml_obs.meta.model_provider',
+
+  INPUT_DOCUMENTS: '_ml_obs.meta.input.documents',
+  INPUT_MESSAGES: '_ml_obs.meta.input.messages',
+  INPUT_VALUE: '_ml_obs.meta.input.value',
+  INPUT_PROMPT: '_ml_obs.meta.input.prompt',
+
+  OUTPUT_DOCUMENTS: '_ml_obs.meta.output.documents',
+  OUTPUT_MESSAGES: '_ml_obs.meta.output.messages',
+  OUTPUT_VALUE: '_ml_obs.meta.output.value',
+
+  INPUT_TOKENS_METRIC_KEY: 'input_tokens',
+  OUTPUT_TOKENS_METRIC_KEY: 'output_tokens',
+  TOTAL_TOKENS_METRIC_KEY: 'total_tokens',
+  CACHE_READ_INPUT_TOKENS_METRIC_KEY: 'cache_read_input_tokens',
+  CACHE_WRITE_INPUT_TOKENS_METRIC_KEY: 'cache_write_input_tokens',
+  CACHE_WRITE_5M_INPUT_TOKENS_METRIC_KEY: 'ephemeral_5m_input_tokens',
+  CACHE_WRITE_1H_INPUT_TOKENS_METRIC_KEY: 'ephemeral_1h_input_tokens',
+  REASONING_OUTPUT_TOKENS_METRIC_KEY: 'reasoning_output_tokens',
+
+  DROPPED_IO_COLLECTION_ERROR: 'dropped_io',
+
+  PROMPT_TRACKING_INSTRUMENTATION_METHOD: 'prompt_tracking_instrumentation_method',
+  PROMPT_MULTIMODAL: 'prompt_multimodal',
+  INSTRUMENTATION_METHOD_AUTO: 'auto',
+  INSTRUMENTATION_METHOD_ANNOTATED: 'annotated',
+  INSTRUMENTATION_METHOD_UNKNOWN: 'unknown',
+
+  ROUTING_API_KEY: '_dd.llmobs.routing.api_key',
+  ROUTING_SITE: '_dd.llmobs.routing.site',
+}

package/packages/dd-trace/src/llmobs/constants/text.js

@@ -0,0 +1,6 @@
+'use strict'
+
+module.exports = {
+  DROPPED_VALUE_TEXT: "[This value has been dropped because this span's size exceeds the 1MB size limit.]",
+  UNSERIALIZABLE_VALUE_TEXT: 'Unserializable value',
+}

package/packages/dd-trace/src/llmobs/constants/writers.js

@@ -0,0 +1,17 @@
+'use strict'
+
+module.exports = {
+  EVP_PROXY_AGENT_BASE_PATH: '/evp_proxy/v2/',
+  EVP_SUBDOMAIN_HEADER_NAME: 'X-Datadog-EVP-Subdomain',
+
+  SPANS_EVENT_TYPE: 'span',
+  SPANS_INTAKE: 'llmobs-intake',
+  SPANS_ENDPOINT: '/api/v2/llmobs',
+
+  EVALUATIONS_INTAKE: 'api',
+  EVALUATIONS_EVENT_TYPE: 'evaluation_metric',
+  EVALUATIONS_ENDPOINT: '/api/intake/llm-obs/v2/eval-metric',
+
+  EVP_PAYLOAD_SIZE_LIMIT: 5 << 20, // 5MB (actual limit is 5.1MB)
+  EVP_EVENT_SIZE_LIMIT: (1 << 20) - 1024, // 999KB (actual limit is 1MB)
+}

package/packages/dd-trace/src/llmobs/index.js

@@ -0,0 +1,149 @@
+'use strict'
+
+const { channel } = require('dc-polyfill')
+
+const log = require('../log')
+const {
+  ML_APP,
+  PROPAGATED_ML_APP_KEY,
+  PROPAGATED_PARENT_ID_KEY,
+} = require('./constants/tags')
+const { storage } = require('./storage')
+const telemetry = require('./telemetry')
+const LLMObsSpanProcessor = require('./span_processor')
+const LLMObsEvalMetricsWriter = require('./writers/evaluations')
+const LLMObsTagger = require('./tagger')
+const LLMObsSpanWriter = require('./writers/spans')
+const { setAgentStrategy } = require('./writers/util')
+
+const spanFinishCh = channel('dd-trace:span:finish')
+const evalMetricAppendCh = channel('llmobs:eval-metric:append')
+const flushCh = channel('llmobs:writers:flush')
+const injectCh = channel('dd-trace:span:inject')
+const registerUserSpanProcessorCh = channel('llmobs:register-processor')
+
+/**
+ * Setting writers and processor globally when LLMObs is enabled
+ * We're setting these in this module instead of on the SDK.
+ * This is to isolate any subscribers and periodic tasks to this module,
+ * and not conditionally instantiate in the SDK, since the SDK is always instantiated
+ * if the tracer is `init`ed. But, in those cases, we don't want to start writers or subscribe
+ * to channels.
+ */
+
+/** @type {LLMObsSpanProcessor | null} */
+let spanProcessor
+
+/** @type {LLMObsSpanWriter | null} */
+let spanWriter
+
+/** @type {LLMObsEvalMetricsWriter | null} */
+let evalWriter
+
+/** @type {import('../config')} */
+let globalTracerConfig
+
+function enable (config) {
+  globalTracerConfig = config
+
+  const startTime = performance.now()
+  // create writers and eval writer append and flush channels
+  // span writer append is handled by the span processor
+  evalWriter = new LLMObsEvalMetricsWriter(config)
+  spanWriter = new LLMObsSpanWriter(config)
+
+  evalMetricAppendCh.subscribe(handleEvalMetricAppend)
+  flushCh.subscribe(handleFlush)
+  registerUserSpanProcessorCh.subscribe(handleRegisterProcessor)
+
+  // span processing
+  spanProcessor = new LLMObsSpanProcessor(config)
+  spanProcessor.setWriter(spanWriter)
+  spanFinishCh.subscribe(handleSpanProcess)
+
+  // distributed tracing for llmobs
+  injectCh.subscribe(handleLLMObsParentIdInjection)
+
+  setAgentStrategy(config, useAgentless => {
+    if (useAgentless && !(config.apiKey && config.site)) {
+      throw new Error(
+        'Cannot send LLM Observability data without a running agent or without both a Datadog API key and site.\n' +
+        'Ensure these configurations are set before running your application.'
+      )
+    }
+
+    evalWriter?.setAgentless(useAgentless)
+    spanWriter?.setAgentless(useAgentless)
+
+    telemetry.recordLLMObsEnabled(startTime, config)
+    log.debug('[LLMObs] Enabled LLM Observability with configuration: %o', config.llmobs)
+  })
+}
+
+function disable () {
+  if (evalMetricAppendCh.hasSubscribers) evalMetricAppendCh.unsubscribe(handleEvalMetricAppend)
+  if (flushCh.hasSubscribers) flushCh.unsubscribe(handleFlush)
+  if (spanFinishCh.hasSubscribers) spanFinishCh.unsubscribe(handleSpanProcess)
+  if (injectCh.hasSubscribers) injectCh.unsubscribe(handleLLMObsParentIdInjection)
+  if (registerUserSpanProcessorCh.hasSubscribers) registerUserSpanProcessorCh.unsubscribe(handleRegisterProcessor)
+
+  spanWriter?.destroy()
+  evalWriter?.destroy()
+  spanProcessor?.setWriter(null)
+
+  spanWriter = null
+  evalWriter = null
+
+  log.debug('[LLMObs] Disabled LLM Observability')
+}
+
+// since LLMObs traces can extend between services and be the same trace,
+// we need to propagate the parent id and mlApp.
+function handleLLMObsParentIdInjection ({ carrier }) {
+  const parent = storage.getStore()?.span
+  const mlObsSpanTags = LLMObsTagger.tagMap.get(parent)
+
+  const parentContext = parent?.context()
+  const parentId = parentContext?.toSpanId()
+  const mlApp =
+    mlObsSpanTags?.[ML_APP] ||
+    parentContext?._trace?.tags?.[PROPAGATED_ML_APP_KEY] ||
+    globalTracerConfig.llmobs.mlApp
+
+  if (parentId) carrier['x-datadog-tags'] += `,${PROPAGATED_PARENT_ID_KEY}=${parentId}`
+  if (mlApp) carrier['x-datadog-tags'] += `,${PROPAGATED_ML_APP_KEY}=${mlApp}`
+}
+
+function handleFlush () {
+  let err = ''
+  try {
+    spanWriter.flush()
+    evalWriter.flush()
+  } catch (e) {
+    err = 'writer_flush_error'
+    log.warn('Failed to flush LLMObs spans and evaluation metrics:', e.message)
+  }
+  telemetry.recordUserFlush(err)
+}
+
+function handleRegisterProcessor (userSpanProcessor) {
+  spanProcessor.setUserSpanProcessor(userSpanProcessor)
+}
+
+function handleSpanProcess (span) {
+  spanProcessor.process(span)
+}
+
+function handleEvalMetricAppend ({ payload, routing }) {
+  try {
+    evalWriter.append(payload, routing)
+  } catch (e) {
+    log.warn(
+      // eslint-disable-next-line @stylistic/max-len
+      'Failed to append evaluation metric to LLM Observability writer, likely due to an unserializable property. Evaluation metrics won\'t be sent to LLM Observability:',
+      e.message
+    )
+  }
+}
+
+module.exports = { enable, disable }

package/packages/dd-trace/src/llmobs/noop.js

@@ -0,0 +1,88 @@
+'use strict'
+
+class NoopLLMObs {
+  constructor (noopTracer) {
+    this._tracer = noopTracer
+  }
+
+  get enabled () {
+    return false
+  }
+
+  enable (options) {}
+
+  disable () {}
+
+  trace (options = {}, fn) {
+    if (typeof options === 'function') {
+      fn = options
+      options = {}
+    }
+
+    const name = options.name || options.kind || fn.name
+
+    return this._tracer.trace(name, options, fn)
+  }
+
+  wrap (options = {}, fn) {
+    if (typeof options === 'function') {
+      fn = options
+      options = {}
+    }
+
+    const name = options.name || options.kind || fn.name
+
+    return this._tracer.wrap(name, options, fn)
+  }
+
+  decorate (options = {}) {
+    const llmobs = this
+    return function (target, ctxOrPropertyKey, descriptor) {
+      if (!ctxOrPropertyKey) return target
+      if (typeof ctxOrPropertyKey === 'object') {
+        const ctx = ctxOrPropertyKey
+        if (ctx.kind !== 'method') return target
+
+        return llmobs.wrap({ name: ctx.name, _decorator: true, ...options }, target)
+      }
+      const propertyKey = ctxOrPropertyKey
+      if (descriptor) {
+        if (typeof descriptor.value !== 'function') return descriptor
+
+        const original = descriptor.value
+        descriptor.value = llmobs.wrap({ name: propertyKey, _decorator: true, ...options }, original)
+
+        return descriptor
+      }
+      if (typeof target[propertyKey] !== 'function') return target[propertyKey]
+
+      const original = target[propertyKey]
+      Object.defineProperty(target, propertyKey, {
+        ...Object.getOwnPropertyDescriptor(target, propertyKey),
+        value: llmobs.wrap({ name: propertyKey, _decorator: true, ...options }, original),
+      })
+
+      return target
+    }
+  }
+
+  annotate (span, options) {}
+
+  exportSpan (span) {
+    return {}
+  }
+
+  submitEvaluation (llmobsSpanContext, options) {}
+
+  flush () {}
+
+  registerProcessor (processor) {}
+
+  deregisterProcessor () {}
+
+  annotationContext (options, fn) { return fn() }
+
+  routingContext (options, fn) { return fn() }
+}
+
+module.exports = NoopLLMObs