dd-trace 4.47.1 → 4.49.0

package/packages/dd-trace/src/llmobs/tagger.js

@@ -0,0 +1,322 @@
+'use strict'
+
+const log = require('../log')
+const {
+  MODEL_NAME,
+  MODEL_PROVIDER,
+  SESSION_ID,
+  ML_APP,
+  SPAN_KIND,
+  INPUT_VALUE,
+  OUTPUT_DOCUMENTS,
+  INPUT_DOCUMENTS,
+  OUTPUT_VALUE,
+  METADATA,
+  METRICS,
+  PARENT_ID_KEY,
+  INPUT_MESSAGES,
+  OUTPUT_MESSAGES,
+  TAGS,
+  NAME,
+  PROPAGATED_PARENT_ID_KEY,
+  ROOT_PARENT_ID,
+  INPUT_TOKENS_METRIC_KEY,
+  OUTPUT_TOKENS_METRIC_KEY,
+  TOTAL_TOKENS_METRIC_KEY
+} = require('./constants/tags')
+
+// global registry of LLMObs spans
+// maps LLMObs spans to their annotations
+const registry = new WeakMap()
+
+class LLMObsTagger {
+  constructor (config, softFail = false) {
+    this._config = config
+
+    this.softFail = softFail
+  }
+
+  static get tagMap () {
+    return registry
+  }
+
+  registerLLMObsSpan (span, {
+    modelName,
+    modelProvider,
+    sessionId,
+    mlApp,
+    parent,
+    kind,
+    name
+  } = {}) {
+    if (!this._config.llmobs.enabled) return
+    if (!kind) return // do not register it in the map if it doesn't have an llmobs span kind
+
+    this._register(span)
+
+    if (name) this._setTag(span, NAME, name)
+
+    this._setTag(span, SPAN_KIND, kind)
+    if (modelName) this._setTag(span, MODEL_NAME, modelName)
+    if (modelProvider) this._setTag(span, MODEL_PROVIDER, modelProvider)
+
+    sessionId = sessionId || parent?.context()._tags[SESSION_ID]
+    if (sessionId) this._setTag(span, SESSION_ID, sessionId)
+
+    if (!mlApp) mlApp = parent?.context()._tags[ML_APP] || this._config.llmobs.mlApp
+    this._setTag(span, ML_APP, mlApp)
+
+    const parentId =
+      parent?.context().toSpanId() ||
+      span.context()._trace.tags[PROPAGATED_PARENT_ID_KEY] ||
+      ROOT_PARENT_ID
+    this._setTag(span, PARENT_ID_KEY, parentId)
+  }
+
+  // TODO: similarly for the following `tag` methods,
+  // how can we transition from a span weakmap to core API functionality
+  tagLLMIO (span, inputData, outputData) {
+    this._tagMessages(span, inputData, INPUT_MESSAGES)
+    this._tagMessages(span, outputData, OUTPUT_MESSAGES)
+  }
+
+  tagEmbeddingIO (span, inputData, outputData) {
+    this._tagDocuments(span, inputData, INPUT_DOCUMENTS)
+    this._tagText(span, outputData, OUTPUT_VALUE)
+  }
+
+  tagRetrievalIO (span, inputData, outputData) {
+    this._tagText(span, inputData, INPUT_VALUE)
+    this._tagDocuments(span, outputData, OUTPUT_DOCUMENTS)
+  }
+
+  tagTextIO (span, inputData, outputData) {
+    this._tagText(span, inputData, INPUT_VALUE)
+    this._tagText(span, outputData, OUTPUT_VALUE)
+  }
+
+  tagMetadata (span, metadata) {
+    this._setTag(span, METADATA, metadata)
+  }
+
+  tagMetrics (span, metrics) {
+    const filterdMetrics = {}
+    for (const [key, value] of Object.entries(metrics)) {
+      let processedKey = key
+
+      // processing these specifically for our metrics ingestion
+      switch (key) {
+        case 'inputTokens':
+          processedKey = INPUT_TOKENS_METRIC_KEY
+          break
+        case 'outputTokens':
+          processedKey = OUTPUT_TOKENS_METRIC_KEY
+          break
+        case 'totalTokens':
+          processedKey = TOTAL_TOKENS_METRIC_KEY
+          break
+      }
+
+      if (typeof value === 'number') {
+        filterdMetrics[processedKey] = value
+      } else {
+        this._handleFailure(`Value for metric '${key}' must be a number, instead got ${value}`)
+      }
+    }
+
+    this._setTag(span, METRICS, filterdMetrics)
+  }
+
+  tagSpanTags (span, tags) {
+    // new tags will be merged with existing tags
+    const currentTags = registry.get(span)?.[TAGS]
+    if (currentTags) {
+      Object.assign(tags, currentTags)
+    }
+    this._setTag(span, TAGS, tags)
+  }
+
+  _tagText (span, data, key) {
+    if (data) {
+      if (typeof data === 'string') {
+        this._setTag(span, key, data)
+      } else {
+        try {
+          this._setTag(span, key, JSON.stringify(data))
+        } catch {
+          const type = key === INPUT_VALUE ? 'input' : 'output'
+          this._handleFailure(`Failed to parse ${type} value, must be JSON serializable.`)
+        }
+      }
+    }
+  }
+
+  _tagDocuments (span, data, key) {
+    if (data) {
+      if (!Array.isArray(data)) {
+        data = [data]
+      }
+
+      const documents = data.map(document => {
+        if (typeof document === 'string') {
+          return { text: document }
+        }
+
+        if (document == null || typeof document !== 'object') {
+          this._handleFailure('Documents must be a string, object, or list of objects.')
+          return undefined
+        }
+
+        const { text, name, id, score } = document
+        let validDocument = true
+
+        if (typeof text !== 'string') {
+          this._handleFailure('Document text must be a string.')
+          validDocument = false
+        }
+
+        const documentObj = { text }
+
+        validDocument = this._tagConditionalString(name, 'Document name', documentObj, 'name') && validDocument
+        validDocument = this._tagConditionalString(id, 'Document ID', documentObj, 'id') && validDocument
+        validDocument = this._tagConditionalNumber(score, 'Document score', documentObj, 'score') && validDocument
+
+        return validDocument ? documentObj : undefined
+      }).filter(doc => !!doc)
+
+      if (documents.length) {
+        this._setTag(span, key, documents)
+      }
+    }
+  }
+
+  _tagMessages (span, data, key) {
+    if (data) {
+      if (!Array.isArray(data)) {
+        data = [data]
+      }
+
+      const messages = data.map(message => {
+        if (typeof message === 'string') {
+          return { content: message }
+        }
+
+        if (message == null || typeof message !== 'object') {
+          this._handleFailure('Messages must be a string, object, or list of objects')
+          return undefined
+        }
+
+        let validMessage = true
+
+        const { content = '', role } = message
+        let toolCalls = message.toolCalls
+        const messageObj = { content }
+
+        if (typeof content !== 'string') {
+          this._handleFailure('Message content must be a string.')
+          validMessage = false
+        }
+
+        validMessage = this._tagConditionalString(role, 'Message role', messageObj, 'role') && validMessage
+
+        if (toolCalls) {
+          if (!Array.isArray(toolCalls)) {
+            toolCalls = [toolCalls]
+          }
+
+          const filteredToolCalls = toolCalls.map(toolCall => {
+            if (typeof toolCall !== 'object') {
+              this._handleFailure('Tool call must be an object.')
+              return undefined
+            }
+
+            let validTool = true
+
+            const { name, arguments: args, toolId, type } = toolCall
+            const toolCallObj = {}
+
+            validTool = this._tagConditionalString(name, 'Tool name', toolCallObj, 'name') && validTool
+            validTool = this._tagConditionalObject(args, 'Tool arguments', toolCallObj, 'arguments') && validTool
+            validTool = this._tagConditionalString(toolId, 'Tool ID', toolCallObj, 'tool_id') && validTool
+            validTool = this._tagConditionalString(type, 'Tool type', toolCallObj, 'type') && validTool
+
+            return validTool ? toolCallObj : undefined
+          }).filter(toolCall => !!toolCall)
+
+          if (filteredToolCalls.length) {
+            messageObj.tool_calls = filteredToolCalls
+          }
+        }
+
+        return validMessage ? messageObj : undefined
+      }).filter(msg => !!msg)
+
+      if (messages.length) {
+        this._setTag(span, key, messages)
+      }
+    }
+  }
+
+  _tagConditionalString (data, type, carrier, key) {
+    if (!data) return true
+    if (typeof data !== 'string') {
+      this._handleFailure(`"${type}" must be a string.`)
+      return false
+    }
+    carrier[key] = data
+    return true
+  }
+
+  _tagConditionalNumber (data, type, carrier, key) {
+    if (!data) return true
+    if (typeof data !== 'number') {
+      this._handleFailure(`"${type}" must be a number.`)
+      return false
+    }
+    carrier[key] = data
+    return true
+  }
+
+  _tagConditionalObject (data, type, carrier, key) {
+    if (!data) return true
+    if (typeof data !== 'object') {
+      this._handleFailure(`"${type}" must be an object.`)
+      return false
+    }
+    carrier[key] = data
+    return true
+  }
+
+  // any public-facing LLMObs APIs using this tagger should not soft fail
+  // auto-instrumentation should soft fail
+  _handleFailure (msg) {
+    if (this.softFail) {
+      log.warn(msg)
+    } else {
+      throw new Error(msg)
+    }
+  }
+
+  _register (span) {
+    if (!this._config.llmobs.enabled) return
+    if (registry.has(span)) {
+      this._handleFailure(`LLMObs Span "${span._name}" already registered.`)
+      return
+    }
+
+    registry.set(span, {})
+  }
+
+  _setTag (span, key, value) {
+    if (!this._config.llmobs.enabled) return
+    if (!registry.has(span)) {
+      this._handleFailure('Span must be an LLMObs generated span.')
+      return
+    }
+
+    const tagsCarrier = registry.get(span)
+    Object.assign(tagsCarrier, { [key]: value })
+  }
+}
+
+module.exports = LLMObsTagger

package/packages/dd-trace/src/llmobs/util.js

@@ -0,0 +1,176 @@
+'use strict'
+
+const { SPAN_KINDS } = require('./constants/tags')
+
+function encodeUnicode (str) {
+  if (!str) return str
+  return str.split('').map(char => {
+    const code = char.charCodeAt(0)
+    if (code > 127) {
+      return `\\u${code.toString(16).padStart(4, '0')}`
+    }
+    return char
+  }).join('')
+}
+
+function validateKind (kind) {
+  if (!SPAN_KINDS.includes(kind)) {
+    throw new Error(`
+      Invalid span kind specified: "${kind}"
+      Must be one of: ${SPAN_KINDS.join(', ')}
+    `)
+  }
+
+  return kind
+}
+
+// extracts the argument names from a function string
+function parseArgumentNames (str) {
+  const result = []
+  let current = ''
+  let closerCount = 0
+  let recording = true
+  let inSingleLineComment = false
+  let inMultiLineComment = false
+
+  for (let i = 0; i < str.length; i++) {
+    const char = str[i]
+    const nextChar = str[i + 1]
+
+    // Handle single-line comments
+    if (!inMultiLineComment && char === '/' && nextChar === '/') {
+      inSingleLineComment = true
+      i++ // Skip the next character
+      continue
+    }
+
+    // Handle multi-line comments
+    if (!inSingleLineComment && char === '/' && nextChar === '*') {
+      inMultiLineComment = true
+      i++ // Skip the next character
+      continue
+    }
+
+    // End of single-line comment
+    if (inSingleLineComment && char === '\n') {
+      inSingleLineComment = false
+      continue
+    }
+
+    // End of multi-line comment
+    if (inMultiLineComment && char === '*' && nextChar === '/') {
+      inMultiLineComment = false
+      i++ // Skip the next character
+      continue
+    }
+
+    // Skip characters inside comments
+    if (inSingleLineComment || inMultiLineComment) {
+      continue
+    }
+
+    if (['{', '[', '('].includes(char)) {
+      closerCount++
+    } else if (['}', ']', ')'].includes(char)) {
+      closerCount--
+    } else if (char === '=' && nextChar !== '>' && closerCount === 0) {
+      recording = false
+      // record the variable name early, and stop counting characters until we reach the next comma
+      result.push(current.trim())
+      current = ''
+      continue
+    } else if (char === ',' && closerCount === 0) {
+      if (recording) {
+        result.push(current.trim())
+        current = ''
+      }
+
+      recording = true
+      continue
+    }
+
+    if (recording) {
+      current += char
+    }
+  }
+
+  if (current && recording) {
+    result.push(current.trim())
+  }
+
+  return result
+}
+
+// finds the bounds of the arguments in a function string
+function findArgumentsBounds (str) {
+  let start = -1
+  let end = -1
+  let closerCount = 0
+
+  for (let i = 0; i < str.length; i++) {
+    const char = str[i]
+
+    if (char === '(') {
+      if (closerCount === 0) {
+        start = i
+      }
+
+      closerCount++
+    } else if (char === ')') {
+      closerCount--
+
+      if (closerCount === 0) {
+        end = i
+        break
+      }
+    }
+  }
+
+  return { start, end }
+}
+
+const memo = new WeakMap()
+function getFunctionArguments (fn, args = []) {
+  if (!fn) return
+  if (!args.length) return
+  if (args.length === 1) return args[0]
+
+  try {
+    let names
+    if (memo.has(fn)) {
+      names = memo.get(fn)
+    } else {
+      const fnString = fn.toString()
+      const { start, end } = findArgumentsBounds(fnString)
+      names = parseArgumentNames(fnString.slice(start + 1, end))
+      memo.set(fn, names)
+    }
+
+    const argsObject = {}
+
+    for (const argIdx in args) {
+      const name = names[argIdx]
+      const arg = args[argIdx]
+
+      const spread = name?.startsWith('...')
+
+      // this can only be the last argument
+      if (spread) {
+        argsObject[name.slice(3)] = args.slice(argIdx)
+        break
+      }
+
+      argsObject[name] = arg
+    }
+
+    return argsObject
+  } catch {
+    return args
+  }
+}
+
+module.exports = {
+  encodeUnicode,
+  validateKind,
+  getFunctionArguments
+}

package/packages/dd-trace/src/llmobs/writers/base.js

@@ -0,0 +1,111 @@
+'use strict'
+
+const request = require('../../exporters/common/request')
+const { URL, format } = require('url')
+
+const logger = require('../../log')
+
+const { encodeUnicode } = require('../util')
+const log = require('../../log')
+
+class BaseLLMObsWriter {
+  constructor ({ interval, timeout, endpoint, intake, eventType, protocol, port }) {
+    this._interval = interval || 1000 // 1s
+    this._timeout = timeout || 5000 // 5s
+    this._eventType = eventType
+
+    this._buffer = []
+    this._bufferLimit = 1000
+    this._bufferSize = 0
+
+    this._url = new URL(format({
+      protocol: protocol || 'https:',
+      hostname: intake,
+      port: port || 443,
+      pathname: endpoint
+    }))
+
+    this._headers = {
+      'Content-Type': 'application/json'
+    }
+
+    this._periodic = setInterval(() => {
+      this.flush()
+    }, this._interval).unref()
+
+    process.once('beforeExit', () => {
+      this.destroy()
+    })
+
+    this._destroyed = false
+
+    logger.debug(`Started ${this.constructor.name} to ${this._url}`)
+  }
+
+  append (event, byteLength) {
+    if (this._buffer.length >= this._bufferLimit) {
+      logger.warn(`${this.constructor.name} event buffer full (limit is ${this._bufferLimit}), dropping event`)
+      return
+    }
+
+    this._bufferSize += byteLength || Buffer.from(JSON.stringify(event)).byteLength
+    this._buffer.push(event)
+  }
+
+  flush () {
+    if (this._buffer.length === 0) {
+      return
+    }
+
+    const events = this._buffer
+    this._buffer = []
+    this._bufferSize = 0
+    const payload = this._encode(this.makePayload(events))
+
+    const options = {
+      headers: this._headers,
+      method: 'POST',
+      url: this._url,
+      timeout: this._timeout
+    }
+
+    log.debug(`Encoded LLMObs payload: ${payload}`)
+
+    request(payload, options, (err, resp, code) => {
+      if (err) {
+        logger.error(
+          `Error sending ${events.length} LLMObs ${this._eventType} events to ${this._url}: ${err.message}`
+        )
+      } else if (code >= 300) {
+        logger.error(
+          `Error sending ${events.length} LLMObs ${this._eventType} events to ${this._url}: ${code}`
+        )
+      } else {
+        logger.debug(`Sent ${events.length} LLMObs ${this._eventType} events to ${this._url}`)
+      }
+    })
+  }
+
+  makePayload (events) {}
+
+  destroy () {
+    if (!this._destroyed) {
+      logger.debug(`Stopping ${this.constructor.name}`)
+      clearInterval(this._periodic)
+      process.removeListener('beforeExit', this.destroy)
+      this.flush()
+      this._destroyed = true
+    }
+  }
+
+  _encode (payload) {
+    return JSON.stringify(payload, (key, value) => {
+      if (typeof value === 'string') {
+        return encodeUnicode(value) // serialize unicode characters
+      }
+      return value
+    }).replace(/\\\\u/g, '\\u') // remove double escaping
+  }
+}
+
+module.exports = BaseLLMObsWriter

package/packages/dd-trace/src/llmobs/writers/evaluations.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const { AGENTLESS_EVALULATIONS_ENDPOINT } = require('../constants/writers')
+const BaseWriter = require('./base')
+
+class LLMObsEvalMetricsWriter extends BaseWriter {
+  constructor (config) {
+    super({
+      endpoint: AGENTLESS_EVALULATIONS_ENDPOINT,
+      intake: `api.${config.site}`,
+      eventType: 'evaluation_metric'
+    })
+
+    this._headers['DD-API-KEY'] = config.apiKey
+  }
+
+  makePayload (events) {
+    return {
+      data: {
+        type: this._eventType,
+        attributes: {
+          metrics: events
+        }
+      }
+    }
+  }
+}
+
+module.exports = LLMObsEvalMetricsWriter

package/packages/dd-trace/src/llmobs/writers/spans/agentProxy.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const {
+  EVP_SUBDOMAIN_HEADER_NAME,
+  EVP_SUBDOMAIN_HEADER_VALUE,
+  EVP_PROXY_AGENT_ENDPOINT
+} = require('../../constants/writers')
+const LLMObsBaseSpanWriter = require('./base')
+
+class LLMObsAgentProxySpanWriter extends LLMObsBaseSpanWriter {
+  constructor (config) {
+    super({
+      intake: config.hostname || 'localhost',
+      protocol: 'http:',
+      endpoint: EVP_PROXY_AGENT_ENDPOINT,
+      port: config.port
+    })
+
+    this._headers[EVP_SUBDOMAIN_HEADER_NAME] = EVP_SUBDOMAIN_HEADER_VALUE
+  }
+}
+
+module.exports = LLMObsAgentProxySpanWriter

package/packages/dd-trace/src/llmobs/writers/spans/agentless.js

@@ -0,0 +1,17 @@
+'use strict'
+
+const { AGENTLESS_SPANS_ENDPOINT } = require('../../constants/writers')
+const LLMObsBaseSpanWriter = require('./base')
+
+class LLMObsAgentlessSpanWriter extends LLMObsBaseSpanWriter {
+  constructor (config) {
+    super({
+      intake: `llmobs-intake.${config.site}`,
+      endpoint: AGENTLESS_SPANS_ENDPOINT
+    })
+
+    this._headers['DD-API-KEY'] = config.apiKey
+  }
+}
+
+module.exports = LLMObsAgentlessSpanWriter