@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/llmobs/telemetry.js

@@ -0,0 +1,174 @@
+'use strict'
+
+const ERROR_TYPE = require('../constants')
+
+const telemetryMetrics = require('../telemetry/metrics')
+const {
+  SPAN_KIND,
+  MODEL_PROVIDER,
+  PARENT_ID_KEY,
+  SESSION_ID,
+  ROOT_PARENT_ID,
+  INTEGRATION,
+  DECORATOR,
+} = require('./constants/tags')
+
+const LLMObsTagger = require('./tagger')
+
+const llmobsMetrics = telemetryMetrics.manager.namespace('mlobs')
+
+function extractIntegrationFromTags (tags) {
+  if (!Array.isArray(tags)) return null
+  const integrationTag = tags.find(tag => tag.startsWith('integration:'))
+  if (!integrationTag) return null
+  return integrationTag.split(':')[1] || null
+}
+
+function extractTagsFromSpanEvent (event) {
+  const spanKind = event.meta?.['span.kind'] || ''
+  const integration = extractIntegrationFromTags(event.tags)
+  const error = event.status === 'error'
+  const autoinstrumented = integration != null
+
+  return {
+    span_kind: spanKind,
+    autoinstrumented: Number(autoinstrumented),
+    error: error ? 1 : 0,
+    integration: integration || 'N/A',
+  }
+}
+
+function incrementLLMObsSpanStartCount (tags, value = 1) {
+  llmobsMetrics.count('span.start', tags).inc(value)
+}
+
+function incrementLLMObsSpanFinishedCount (span, value = 1) {
+  const mlObsTags = LLMObsTagger.tagMap.get(span)
+  const spanTags = span.context()._tags
+
+  const isRootSpan = mlObsTags[PARENT_ID_KEY] === ROOT_PARENT_ID
+  const hasSessionId = mlObsTags[SESSION_ID] != null
+  const integration = mlObsTags[INTEGRATION]
+  const autoInstrumented = integration != null
+  const decorator = !!mlObsTags[DECORATOR]
+  const spanKind = mlObsTags[SPAN_KIND]
+  const modelProvider = mlObsTags[MODEL_PROVIDER]
+  const error = spanTags.error || spanTags[ERROR_TYPE]
+
+  const tags = {
+    autoinstrumented: Number(autoInstrumented),
+    has_session_id: Number(hasSessionId),
+    is_root_span: Number(isRootSpan),
+    span_kind: spanKind,
+    integration: integration || 'N/A',
+    error: error ? 1 : 0,
+  }
+  if (!autoInstrumented) {
+    tags.decorator = Number(decorator)
+  }
+  if (modelProvider) {
+    tags.model_provider = modelProvider
+  }
+
+  llmobsMetrics.count('span.finished', tags).inc(value)
+}
+
+function recordLLMObsEnabled (startTime, config, value = 1) {
+  const initTimeMs = performance.now() - startTime
+  // There isn't an easy way to determine if a user automatically enabled LLMObs via
+  // in-code or command line setup. We'll use the presence of DD_LLMOBS_ENABLED env var
+  // as a rough heuristic, but note that this isn't perfect since
+  // a user may have env vars but enable manually in code.
+  const autoEnabled = !!config._env?.['llmobs.enabled']
+  const tags = {
+    error: 0,
+    agentless: Number(config.llmobs.agentlessEnabled),
+    site: config.site,
+    auto: Number(autoEnabled),
+    ml_app: config.llmobs.mlApp,
+  }
+  llmobsMetrics.count('product_enabled', tags).inc(value)
+  llmobsMetrics.distribution('init_time', tags).track(initTimeMs)
+}
+
+function recordLLMObsRawSpanSize (event, rawEventSize) {
+  const tags = extractTagsFromSpanEvent(event)
+  llmobsMetrics.distribution('span.raw_size', tags).track(rawEventSize)
+}
+
+function recordLLMObsSpanSize (event, eventSize, shouldTruncate) {
+  const tags = extractTagsFromSpanEvent(event)
+  tags.truncated = Number(shouldTruncate)
+  llmobsMetrics.distribution('span.size', tags).track(eventSize)
+}
+
+function recordDroppedPayload (numEvents, eventType, error) {
+  if (eventType !== 'span' && eventType !== 'evaluation_metric') return
+  const metricName = eventType === 'span' ? 'dropped_span_event' : 'dropped_eval_event'
+  const tags = { error }
+  llmobsMetrics.count(metricName, tags).inc(numEvents)
+}
+
+function recordLLMObsAnnotate (span, err, value = 1) {
+  const mlObsTags = LLMObsTagger.tagMap.get(span) || {}
+  const spanKind = mlObsTags[SPAN_KIND] || 'N/A'
+  const isRootSpan = mlObsTags[PARENT_ID_KEY] === ROOT_PARENT_ID
+
+  const tags = {
+    error: Number(!!err),
+    span_kind: spanKind,
+    is_root_span: Number(isRootSpan),
+  }
+  if (err) tags.error_type = err
+  llmobsMetrics.count('annotations', tags).inc(value)
+}
+
+function recordUserFlush (err, value = 1) {
+  const tags = { error: Number(!!err) }
+  if (err) tags.error_type = err
+  llmobsMetrics.count('user_flush', tags).inc(value)
+}
+
+function recordExportSpan (span, err, value = 1) {
+  const mlObsTags = LLMObsTagger.tagMap.get(span) || {}
+  const spanKind = mlObsTags[SPAN_KIND] || 'N/A'
+  const isRootSpan = mlObsTags[PARENT_ID_KEY] === ROOT_PARENT_ID
+
+  const tags = {
+    error: Number(!!err),
+    span_kind: spanKind,
+    is_root_span: Number(isRootSpan),
+  }
+  if (err) tags.error_type = err
+  llmobsMetrics.count('spans_exported', tags).inc(value)
+}
+
+function recordSubmitEvaluation (options, err, value = 1) {
+  const tags = {
+    error: Number(!!err),
+    custom_joining_key: 0,
+  }
+  const metricType = options?.metricType?.toLowerCase()
+  if (metricType !== 'categorical' && metricType !== 'score') tags.metric_type = 'other'
+  if (err) tags.error_type = err
+  llmobsMetrics.count('evals_submitted', tags).inc(value)
+}
+
+function recordLLMObsUserProcessorCalled (error, value = 1) {
+  const tags = { error: error ? 1 : 0 }
+  llmobsMetrics.count('user_processor_called', tags).inc(value)
+}
+
+module.exports = {
+  recordLLMObsEnabled,
+  incrementLLMObsSpanStartCount,
+  incrementLLMObsSpanFinishedCount,
+  recordLLMObsRawSpanSize,
+  recordLLMObsSpanSize,
+  recordDroppedPayload,
+  recordLLMObsAnnotate,
+  recordUserFlush,
+  recordExportSpan,
+  recordSubmitEvaluation,
+  recordLLMObsUserProcessorCalled,
+}

package/packages/dd-trace/src/llmobs/util.js

@@ -0,0 +1,182 @@
+'use strict'
+
+const { SPAN_KINDS } = require('./constants/tags')
+
+function encodeUnicode (str = '') {
+  let result = ''
+  for (let i = 0; i < str.length; i++) {
+    const code = str.charCodeAt(i)
+    result += code > 127 ? String.raw`\u${code.toString(16).padStart(4, '0')}` : str[i]
+  }
+  return result
+}
+
+function validateKind (kind) {
+  if (!SPAN_KINDS.includes(kind)) {
+    throw new Error(`
+      Invalid span kind specified: "${kind}"
+      Must be one of: ${SPAN_KINDS.join(', ')}
+    `)
+  }
+
+  return kind
+}
+
+// extracts the argument names from a function string
+function parseArgumentNames (str) {
+  const result = []
+  let current = ''
+  let closerCount = 0
+  let recording = true
+  let inSingleLineComment = false
+  let inMultiLineComment = false
+
+  for (let i = 0; i < str.length; i++) {
+    const char = str[i]
+    const nextChar = str[i + 1]
+
+    // Handle single-line comments
+    if (!inMultiLineComment && char === '/' && nextChar === '/') {
+      inSingleLineComment = true
+      i++ // Skip the next character
+      continue
+    }
+
+    // Handle multi-line comments
+    if (!inSingleLineComment && char === '/' && nextChar === '*') {
+      inMultiLineComment = true
+      i++ // Skip the next character
+      continue
+    }
+
+    // End of single-line comment
+    if (inSingleLineComment && char === '\n') {
+      inSingleLineComment = false
+      continue
+    }
+
+    // End of multi-line comment
+    if (inMultiLineComment && char === '*' && nextChar === '/') {
+      inMultiLineComment = false
+      i++ // Skip the next character
+      continue
+    }
+
+    // Skip characters inside comments
+    if (inSingleLineComment || inMultiLineComment) {
+      continue
+    }
+
+    if (['{', '[', '('].includes(char)) {
+      closerCount++
+    } else if (['}', ']', ')'].includes(char)) {
+      closerCount--
+    } else if (char === '=' && nextChar !== '>' && closerCount === 0) {
+      recording = false
+      // record the variable name early, and stop counting characters until we reach the next comma
+      result.push(current.trim())
+      current = ''
+      continue
+    } else if (char === ',' && closerCount === 0) {
+      if (recording) {
+        result.push(current.trim())
+        current = ''
+      }
+
+      recording = true
+      continue
+    }
+
+    if (recording) {
+      current += char
+    }
+  }
+
+  if (current && recording) {
+    result.push(current.trim())
+  }
+
+  return result
+}
+
+// finds the bounds of the arguments in a function string
+function findArgumentsBounds (str) {
+  let start = -1
+  let end = -1
+  let closerCount = 0
+
+  // TODO(BridgeAR): This "breaks" up codePoints.
+  // Investigate if this is a problem.
+  for (let i = 0; i < str.length; i++) {
+    const char = str[i]
+
+    if (char === '(') {
+      if (closerCount === 0) {
+        start = i
+      }
+
+      closerCount++
+    } else if (char === ')') {
+      closerCount--
+
+      if (closerCount === 0) {
+        end = i
+        break
+      }
+    }
+  }
+
+  return { start, end }
+}
+
+const memo = new WeakMap()
+function getFunctionArguments (fn, args = []) {
+  if (!fn) return
+  if (!args.length) return
+  if (args.length === 1) return args[0]
+
+  try {
+    let names
+    if (memo.has(fn)) {
+      names = memo.get(fn)
+    } else {
+      const fnString = fn.toString()
+      const { start, end } = findArgumentsBounds(fnString)
+      names = parseArgumentNames(fnString.slice(start + 1, end))
+      memo.set(fn, names)
+    }
+
+    const argsObject = {}
+
+    for (const argIdx in args) {
+      const name = names[argIdx]
+      const arg = args[argIdx]
+
+      const spread = name?.startsWith('...')
+
+      // this can only be the last argument
+      if (spread) {
+        argsObject[name.slice(3)] = args.slice(argIdx)
+        break
+      }
+
+      argsObject[name] = arg
+    }
+
+    return argsObject
+  } catch {
+    return args
+  }
+}
+
+function spanHasError (span) {
+  const tags = span.context()._tags
+  return !!(tags.error || tags['error.type'])
+}
+
+module.exports = {
+  encodeUnicode,
+  validateKind,
+  getFunctionArguments,
+  spanHasError,
+}

package/packages/dd-trace/src/llmobs/writers/base.js

@@ -0,0 +1,242 @@
+'use strict'
+
+const { URL, format } = require('node:url')
+const path = require('node:path')
+const request = require('../../exporters/common/request')
+const { getEnvironmentVariable } = require('../../config/helper')
+
+const logger = require('../../log')
+
+const { encodeUnicode } = require('../util')
+const telemetry = require('../telemetry')
+const log = require('../../log')
+const {
+  EVP_SUBDOMAIN_HEADER_NAME,
+  EVP_PROXY_AGENT_BASE_PATH,
+} = require('../constants/writers')
+const { getAgentUrl } = require('../../agent/url')
+const { parseResponseAndLog } = require('./util')
+
+class LLMObsBuffer {
+  constructor ({ events, size, routing = {}, isDefault = false, limit = 1000 }) {
+    this.events = events
+    this.size = size
+    this.routing = routing
+    this.isDefault = isDefault
+    this.limit = limit
+  }
+
+  clear () {
+    this.events = []
+    this.size = 0
+  }
+}
+
+class BaseLLMObsWriter {
+  #destroyer
+  /** @type {Map<string, LLMObsBuffer>} */
+  #multiTenantBuffers = new Map()
+
+  constructor ({ interval, timeout, eventType, config, endpoint, intake }) {
+    this._interval = interval ?? getEnvironmentVariable('_DD_LLMOBS_FLUSH_INTERVAL') ?? 1000 // 1s
+    this._timeout = timeout ?? getEnvironmentVariable('_DD_LLMOBS_TIMEOUT') ?? 5000 // 5s
+    this._eventType = eventType
+
+    /** @type {LLMObsBuffer} */
+    this._buffer = new LLMObsBuffer({ events: [], size: 0, isDefault: true })
+
+    this._config = config
+    this._endpoint = endpoint
+    this._baseEndpoint = endpoint // should not be unset
+    this._intake = intake
+
+    this._periodic = setInterval(() => {
+      this.flush()
+    }, this._interval).unref()
+
+    const destroyer = this.destroy.bind(this)
+    globalThis[Symbol.for('dd-trace')].beforeExitHandlers.add(destroyer)
+
+    this.#destroyer = destroyer
+  }
+
+  // Split on protocol separator to preserve it
+  // path.join will remove some slashes unnecessarily
+  #buildUrl (baseUrl, endpoint) {
+    const [protocol, rest] = baseUrl.split('://')
+    return protocol + '://' + path.join(rest, endpoint)
+  }
+
+  get url () {
+    if (this._agentless == null) return null
+    return this.#buildUrl(this._baseUrl.href, this._endpoint)
+  }
+
+  _getBuffer (routing) {
+    if (!routing?.apiKey) {
+      return this._buffer
+    }
+    const apiKey = routing.apiKey
+    let buffer = this.#multiTenantBuffers.get(apiKey)
+    if (!buffer) {
+      buffer = new LLMObsBuffer({ events: [], size: 0, routing })
+      this.#multiTenantBuffers.set(apiKey, buffer)
+    }
+    return buffer
+  }
+
+  append (event, routing, byteLength) {
+    const buffer = this._getBuffer(routing)
+
+    if (buffer.events.length >= buffer.limit) {
+      logger.warn(`${this.constructor.name} event buffer full (limit is ${buffer.limit}), dropping event`)
+      telemetry.recordDroppedPayload(1, this._eventType, 'buffer_full')
+      return
+    }
+
+    const eventSize = byteLength || Buffer.byteLength(JSON.stringify(event))
+
+    buffer.size += eventSize
+    buffer.events.push(event)
+  }
+
+  flush () {
+    if (this._agentless == null) {
+      return
+    }
+
+    // Flush default buffer
+    if (this._buffer.events.length > 0) {
+      const events = this._buffer.events
+      this._buffer.clear()
+
+      const payload = this._encode(this.makePayload(events))
+
+      log.debug('Encoded LLMObs payload: %s', payload)
+
+      const options = this._getOptions()
+
+      request(payload, options, (err, resp, code) => {
+        parseResponseAndLog(err, code, events.length, this.url, this._eventType)
+      })
+    }
+
+    // Flush multi-tenant buffers
+    for (const [apiKey, buffer] of this.#multiTenantBuffers) {
+      if (buffer.events.length === 0) continue
+
+      const events = buffer.events
+      buffer.clear()
+
+      const payload = this._encode(this.makePayload(events))
+      const site = buffer.routing.site || this._config.site
+      const options = {
+        headers: {
+          'Content-Type': 'application/json',
+          'DD-API-KEY': apiKey,
+        },
+        method: 'POST',
+        timeout: this._timeout,
+        url: new URL(format({
+          protocol: 'https:',
+          hostname: `${this._intake}.${site}`,
+        })),
+        path: this._baseEndpoint,
+      }
+      const url = this.#buildUrl(options.url.href, options.path)
+      const maskedApiKey = apiKey ? `****${apiKey.slice(-4)}` : ''
+
+      log.debug('Encoding and flushing multi-tenant buffer for %s', maskedApiKey)
+      log.debug('Encoded LLMObs payload: %s', payload)
+
+      request(payload, options, (err, resp, code) => {
+        parseResponseAndLog(err, code, events.length, url, this._eventType)
+      })
+    }
+
+    this.#cleanupEmptyBuffers()
+  }
+
+  #cleanupEmptyBuffers () {
+    for (const [key, buffer] of this.#multiTenantBuffers) {
+      if (buffer.events.length === 0) {
+        this.#multiTenantBuffers.delete(key)
+      }
+    }
+  }
+
+  makePayload (events) {}
+
+  destroy () {
+    if (this.#destroyer) {
+      logger.debug(`Stopping ${this.constructor.name}`)
+      clearInterval(this._periodic)
+      globalThis[Symbol.for('dd-trace')].beforeExitHandlers.delete(this.#destroyer)
+      this.flush()
+      this.#destroyer = undefined
+    }
+  }
+
+  setAgentless (agentless) {
+    this._agentless = agentless
+    const { url, endpoint } = this._getUrlAndPath()
+
+    this._baseUrl = url
+    this._endpoint = endpoint
+
+    logger.debug(`Configuring ${this.constructor.name} to ${this.url}`)
+  }
+
+  _getUrlAndPath () {
+    if (this._agentless) {
+      const site = this._config.site
+      return {
+        url: new URL(format({
+          protocol: 'https:',
+          hostname: `${this._intake}.${site}`,
+        })),
+        endpoint: this._endpoint,
+      }
+    }
+
+    const overrideOriginEnv = getEnvironmentVariable('_DD_LLMOBS_OVERRIDE_ORIGIN')
+    const overrideOriginUrl = overrideOriginEnv && new URL(overrideOriginEnv)
+    const base = overrideOriginUrl ?? getAgentUrl(this._config)
+
+    return {
+      url: base,
+      endpoint: path.join(EVP_PROXY_AGENT_BASE_PATH, this._endpoint),
+    }
+  }
+
+  _getOptions () {
+    const options = {
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      method: 'POST',
+      timeout: this._timeout,
+      url: this._baseUrl,
+      path: this._endpoint,
+    }
+
+    if (this._agentless) {
+      options.headers['DD-API-KEY'] = this._config.apiKey || ''
+    } else {
+      options.headers[EVP_SUBDOMAIN_HEADER_NAME] = this._intake
+    }
+
+    return options
+  }
+
+  _encode (payload) {
+    return JSON.stringify(payload, (key, value) => {
+      if (typeof value === 'string') {
+        return encodeUnicode(value) // serialize unicode characters
+      }
+      return value
+    }).replaceAll(String.raw`\\u`, String.raw`\u`) // remove double escaping
+  }
+}
+
+module.exports = BaseLLMObsWriter

package/packages/dd-trace/src/llmobs/writers/evaluations.js

@@ -0,0 +1,32 @@
+'use strict'
+
+const {
+  EVALUATIONS_ENDPOINT,
+  EVALUATIONS_EVENT_TYPE,
+  EVALUATIONS_INTAKE,
+} = require('../constants/writers')
+const BaseWriter = require('./base')
+
+class LLMObsEvalMetricsWriter extends BaseWriter {
+  constructor (config) {
+    super({
+      config,
+      intake: EVALUATIONS_INTAKE,
+      eventType: EVALUATIONS_EVENT_TYPE,
+      endpoint: EVALUATIONS_ENDPOINT,
+    })
+  }
+
+  makePayload (events) {
+    return {
+      data: {
+        type: this._eventType,
+        attributes: {
+          metrics: events,
+        },
+      },
+    }
+  }
+}
+
+module.exports = LLMObsEvalMetricsWriter

package/packages/dd-trace/src/llmobs/writers/spans.js

@@ -0,0 +1,69 @@
+'use strict'
+
+const {
+  EVP_EVENT_SIZE_LIMIT,
+  EVP_PAYLOAD_SIZE_LIMIT,
+  SPANS_ENDPOINT,
+  SPANS_EVENT_TYPE,
+  SPANS_INTAKE,
+} = require('../constants/writers')
+const { DROPPED_VALUE_TEXT } = require('../constants/text')
+const { DROPPED_IO_COLLECTION_ERROR } = require('../constants/tags')
+const telemetry = require('../telemetry')
+const logger = require('../../log')
+const tracerVersion = require('../../../../../package.json').version
+const BaseWriter = require('./base')
+
+class LLMObsSpanWriter extends BaseWriter {
+  constructor (config) {
+    super({
+      config,
+      eventType: SPANS_EVENT_TYPE,
+      intake: SPANS_INTAKE,
+      endpoint: SPANS_ENDPOINT,
+    })
+  }
+
+  append (event, routing) {
+    const eventSizeBytes = Buffer.byteLength(JSON.stringify(event))
+    telemetry.recordLLMObsRawSpanSize(event, eventSizeBytes)
+
+    const shouldTruncate = eventSizeBytes > EVP_EVENT_SIZE_LIMIT
+    let processedEventSizeBytes = eventSizeBytes
+
+    if (shouldTruncate) {
+      logger.warn(`Dropping event input/output because its size (${eventSizeBytes}) exceeds the 1MB event size limit`)
+      event = this._truncateSpanEvent(event)
+      processedEventSizeBytes = Buffer.byteLength(JSON.stringify(event))
+    }
+
+    telemetry.recordLLMObsSpanSize(event, processedEventSizeBytes, shouldTruncate)
+
+    const buffer = this._getBuffer(routing)
+    if (buffer.size + processedEventSizeBytes > EVP_PAYLOAD_SIZE_LIMIT) {
+      logger.debug('Flushing queue because queuing next event will exceed EvP payload limit')
+      this.flush()
+    }
+
+    super.append(event, routing, processedEventSizeBytes)
+  }
+
+  makePayload (events) {
+    return events.map(event => ({
+      '_dd.stage': 'raw',
+      '_dd.tracer_version': tracerVersion,
+      event_type: this._eventType,
+      spans: [event],
+    }))
+  }
+
+  _truncateSpanEvent (event) {
+    event.meta.input = { value: DROPPED_VALUE_TEXT }
+    event.meta.output = { value: DROPPED_VALUE_TEXT }
+
+    event.collection_errors = [DROPPED_IO_COLLECTION_ERROR]
+    return event
+  }
+}
+
+module.exports = LLMObsSpanWriter