dd-trace 6.16.0 → 6.17.0

package/packages/dd-trace/src/llmobs/plugins/openai/realtime.js

@@ -0,0 +1,285 @@
+'use strict'
+
+const {
+  g711ToPcm16,
+  g711Variant,
+  isPcm16AudioMime,
+  pcm16ToWav,
+} = require('../../audio-codec')
+const { fitsInlineAudioBudget, formatAudioPartWithGuard } = require('../../audio-utils')
+const {
+  AUDIO_FALLBACK,
+  G711_SAMPLE_RATE,
+  LLMOBS_AUDIO_INLINE_MAX_BYTES,
+  PCM16_BYTES_PER_SAMPLE,
+  WAV_HEADER_BYTES,
+} = require('../../constants/audio')
+const { storage: llmobsStorage } = require('../../storage')
+const { safeJsonParse } = require('../../util')
+const LLMObsPlugin = require('../base')
+const { getModelProviderAndClient } = require('./utils')
+
+/**
+ * @typedef {import('../../audio-utils').AudioPart} AudioPart
+ * @typedef {{
+ *   role: string,
+ *   content: string,
+ *   audioParts?: AudioPart[],
+ *   toolCalls?: object[],
+ *   toolResults?: object[],
+ * }} Message
+ */
+
+/**
+ * Turn a turn side's raw audio into a playable audio part.
+ *
+ * Realtime audio is raw PCM16 (24 kHz mono) by default, which the UI can't render, so it is wrapped
+ * in a WAV container — lossless, just a header. G.711 telephony audio, used by phone-call
+ * integrations, is decoded to PCM16 and likewise WAV-wrapped at its fixed 8 kHz rate. Anything else
+ * is passed to the size guard as-is, which keeps it only if it is renderable and within budget.
+ *
+ * Both conversions expand what they are given — G.711 doubles it, and either way a WAV header is
+ * prepended — so the budget is checked against the size the conversion *would* produce before
+ * running it. Checking afterwards, as the guard alone does, means decoding and copying megabytes
+ * only to discard them: the retention cap admits roughly twice what a G.711 turn can ever inline.
+ *
+ * @param {Buffer} audio
+ * @param {string} mimeType
+ * @param {number} sampleRate
+ * @param {number} maxBytes - Encoded bytes still available to this turn. See `setLLMObsTags`.
+ * @returns {AudioPart | undefined}
+ */
+function buildAudioPart (audio, mimeType, sampleRate, maxBytes) {
+  if (!audio.length) return
+
+  if (isPcm16AudioMime(mimeType)) {
+    if (!fitsInlineAudioBudget(audio.length + WAV_HEADER_BYTES, maxBytes)) return
+    return formatAudioPartWithGuard(pcm16ToWav(audio, sampleRate), 'audio/wav', maxBytes)
+  }
+
+  const variant = g711Variant(mimeType)
+  if (variant !== undefined) {
+    if (!fitsInlineAudioBudget(audio.length * PCM16_BYTES_PER_SAMPLE + WAV_HEADER_BYTES, maxBytes)) return
+    return formatAudioPartWithGuard(
+      pcm16ToWav(g711ToPcm16(audio, variant), G711_SAMPLE_RATE), 'audio/wav', maxBytes
+    )
+  }
+
+  return formatAudioPartWithGuard(audio, mimeType, maxBytes)
+}
+
+/**
+ * Encoded size an already-built message contributes to the span event, which for audio is the
+ * base64 content itself.
+ *
+ * @param {Message | undefined} message
+ */
+function audioBytesOf (message) {
+  return message?.audioParts?.[0]?.content?.length ?? 0
+}
+
+/**
+ * @param {object[]} toolCalls
+ * @returns {object[]}
+ */
+function parseToolCallArguments (toolCalls) {
+  return toolCalls.map(toolCall => ({ ...toolCall, arguments: safeJsonParse(toolCall.arguments ?? '', {}) }))
+}
+
+/**
+ * Build the message for one side of a turn.
+ *
+ * @param {string} role
+ * @param {import('../../../../../datadog-instrumentations/src/openai-realtime/session').TurnSide} side
+ * @param {number} maxAudioBytes - Encoded audio bytes still available to this turn.
+ * @returns {Message | undefined}
+ */
+function buildMessage (role, side, maxAudioBytes) {
+  const audioPart = buildAudioPart(side.audio, side.mimeType, side.sampleRate, maxAudioBytes)
+
+  let content = side.transcript || side.text
+  if (!content && audioPart === undefined && side.audioPresent) {
+    // Audio was captured but couldn't be turned into a playable part — an unsupported format, or
+    // over the size budget — and there is no transcript, so surface a marker rather than let the
+    // turn look silently empty.
+    content = AUDIO_FALLBACK
+  }
+
+  const toolCalls = side.toolCalls?.length ? parseToolCallArguments(side.toolCalls) : undefined
+  const toolResults = side.toolResults.length ? side.toolResults : undefined
+
+  if (!content && audioPart === undefined && toolCalls === undefined && toolResults === undefined) return
+
+  /** @type {Message} */
+  const message = { role, content: content || '' }
+  if (audioPart !== undefined) message.audioParts = [audioPart]
+  if (toolCalls !== undefined) message.toolCalls = toolCalls
+  if (toolResults !== undefined) message.toolResults = toolResults
+
+  return message
+}
+
+/**
+ * @param {{ input_tokens?: number, output_tokens?: number, total_tokens?: number } | undefined} usage
+ * @returns {Record<string, number> | undefined}
+ */
+function usageMetrics (usage) {
+  if (!usage) return
+
+  const { input_tokens: inputTokens, output_tokens: outputTokens } = usage
+  let { total_tokens: totalTokens } = usage
+
+  /** @type {Record<string, number>} */
+  const metrics = {}
+  if (inputTokens != null) metrics.input_tokens = inputTokens
+  if (outputTokens != null) metrics.output_tokens = outputTokens
+  // Mirror the chat and responses fallback.
+  if (totalTokens == null && inputTokens != null && outputTokens != null) {
+    totalTokens = inputTokens + outputTokens
+  }
+  if (totalTokens != null) metrics.total_tokens = totalTokens
+
+  return metrics.total_tokens === undefined && metrics.input_tokens === undefined ? undefined : metrics
+}
+
+/**
+ * LLM Observability spans for one OpenAI Realtime turn.
+ *
+ * The turn is modelled as a small tree — a `workflow` root with a `user speech` window, the `llm`
+ * generation span, and an `agent speech` window — so span duration is meaningful (the llm span
+ * measures model work, not the human's speaking time) and time-to-first-agent-audio falls out of the
+ * span boundaries. Consumers identify the phase spans by span kind and name, so those strings are a
+ * contract: renaming one is a breaking change for the UI and the backend metric alike.
+ */
+class RealtimeLLMObsPlugin extends LLMObsPlugin {
+  static integration = 'openai'
+  static system = 'openai'
+
+  /**
+   * The instrumentation replays the turn with `traceSync`, which never publishes `asyncEnd`, so tag
+   * on `end` instead — before the sibling tracing plugin's `end` finishes the span, since the LLM
+   * Observability plugins are registered first.
+   *
+   * @param {object} ctx
+   */
+  end (ctx) {
+    super.end(ctx)
+    super.asyncEnd(ctx)
+  }
+}
+
+/** The whole perceived turn. Carries the transcripts so the waterfall row reads as a conversation. */
+class RealtimeTurnLLMObsPlugin extends RealtimeLLMObsPlugin {
+  static id = 'openai_realtime_turn_llmobs'
+  static prefix = 'tracing:apm:openai:realtime:turn'
+
+  constructor (...args) {
+    super(...args)
+
+    this.addSub('dd-trace:openai:realtime:capture-context', turn => {
+      if (!this._tracerConfig.llmobs.DD_LLMOBS_ENABLED) return
+
+      // Composed on top of the tracing plugin's, so the replayed tree restores both the APM and the
+      // LLM Observability context the caller had active when the turn began.
+      const store = llmobsStorage.getStore()
+      const previous = turn.runInContext ?? (fn => fn())
+      turn.runInContext = fn => previous(() => llmobsStorage.run(store, fn))
+    })
+  }
+
+  getLLMObsSpanRegisterOptions (ctx) {
+    return {
+      kind: 'workflow',
+      // Provider-agnostic `audio turn` marker, so a consumer can gate on "is this a voice turn?".
+      name: 'realtime audio turn',
+      sessionId: ctx.turn.sessionId,
+    }
+  }
+
+  setLLMObsTags (ctx) {
+    const span = ctx.currentStore?.span
+    if (!span) return
+
+    const { input, output } = ctx.turn
+    this._tagger.tagTextIO(span, input.transcript || input.text, output.transcript || output.text)
+  }
+}
+
+/** The model's generation work. The turn's audio, tool calls and token usage ride here. */
+class RealtimeResponseLLMObsPlugin extends RealtimeLLMObsPlugin {
+  static id = 'openai_realtime_response_llmobs'
+  static prefix = 'tracing:apm:openai:realtime:response'
+
+  constructor (...args) {
+    super(...args)
+
+    // This plugin is the only consumer of a turn's audio bytes, and the instrumentation keeps them
+    // only while something is subscribed here — a disabled plugin unsubscribes, so an APM-only
+    // process never buffers audio it cannot use. Nothing is ever published; the subscription is the
+    // signal. Keep it if `buildMessage` stops reading `side.audio`, and the buffering stops with it.
+    this.addSub('dd-trace:openai:realtime:audio', () => {})
+  }
+
+  getLLMObsSpanRegisterOptions (ctx) {
+    const { turn } = ctx
+    const { modelProvider, client } = getModelProviderAndClient(turn.basePath)
+
+    return {
+      kind: 'llm',
+      name: `${client}.createRealtimeResponse`,
+      modelName: turn.model || 'unknown_model',
+      modelProvider,
+      sessionId: turn.sessionId,
+    }
+  }
+
+  setLLMObsTags (ctx) {
+    const span = ctx.currentStore?.span
+    if (!span) return
+
+    const { turn } = ctx
+
+    // One budget for the whole event, spent across both sides rather than offered to each.
+    // A turn normally carries audio on both sides, so a per-message budget lets two individually
+    // accepted clips add up to twice the limit — and `writers/spans.js` responds by truncating the
+    // event's *entire* input and output, losing the transcripts too. Input is served first and the
+    // output falls back to its transcript when nothing is left, which is the same graceful
+    // degradation an oversize single clip already gets.
+    const inputMessage = buildMessage('user', turn.input, LLMOBS_AUDIO_INLINE_MAX_BYTES)
+    const remainingAudioBytes = LLMOBS_AUDIO_INLINE_MAX_BYTES - audioBytesOf(inputMessage)
+    const outputMessage = buildMessage('assistant', turn.output, remainingAudioBytes)
+
+    this._tagger.tagLLMIO(span, inputMessage ? [inputMessage] : [], outputMessage ? [outputMessage] : [])
+    this._tagger.tagMetadata(span, turn.metadata)
+
+    const metrics = usageMetrics(turn.usage)
+    if (metrics !== undefined) this._tagger.tagMetrics(span, metrics)
+  }
+}
+
+/** A speaking window — the human's or the agent's. These are timing regions; the audio rides on the llm span. */
+class RealtimeSpeechLLMObsPlugin extends RealtimeLLMObsPlugin {
+  static id = 'openai_realtime_speech_llmobs'
+  static prefix = 'tracing:apm:openai:realtime:speech'
+
+  getLLMObsSpanRegisterOptions (ctx) {
+    return {
+      kind: 'workflow',
+      name: ctx.phase.llmobsName,
+      sessionId: ctx.turn.sessionId,
+    }
+  }
+
+  setLLMObsTags (ctx) {
+    const span = ctx.currentStore?.span
+    if (!span) return
+
+    this._tagger.tagTextIO(span, undefined, ctx.transcript)
+  }
+}
+
+module.exports = [
+  RealtimeTurnLLMObsPlugin,
+  RealtimeResponseLLMObsPlugin,
+  RealtimeSpeechLLMObsPlugin,
+]

package/packages/dd-trace/src/llmobs/plugins/openai/utils.js

@@ -1,13 +1,13 @@
 'use strict'
 
+const { audioMimeTypeFromFormat, formatAudioPart } = require('../../audio-utils')
+const { AUDIO_FALLBACK } = require('../../constants/audio')
 const { UNKNOWN_MODEL_PROVIDER } = require('../../constants/tags')
-const { audioMimeTypeFromFormat, formatAudioPart } = require('../../util')
 const {
   INPUT_TYPE_IMAGE,
   INPUT_TYPE_FILE,
   IMAGE_FALLBACK,
   FILE_FALLBACK,
-  AUDIO_FALLBACK,
   AUDIO_MIME_TYPES,
 } = require('./constants')
 
@@ -177,7 +177,6 @@ function extractContentParts (parts) {
  * client baseURL convention.
  *
  * @param {string} baseUrl
- * @returns {string}
  */
 function getOpenAIModelProvider (baseUrl = '') {
   if (baseUrl.includes('azure')) return 'azure_openai'
@@ -186,11 +185,25 @@ function getOpenAIModelProvider (baseUrl = '') {
   return UNKNOWN_MODEL_PROVIDER
 }
 
+/**
+ * Pair an OpenAI-compatible base URL's model provider with the client label used in span names.
+ *
+ * @param {string} [baseUrl]
+ * @returns {{ modelProvider: string, client: string }}
+ */
+function getModelProviderAndClient (baseUrl = '') {
+  const modelProvider = getOpenAIModelProvider(baseUrl)
+  if (modelProvider === 'azure_openai') return { modelProvider, client: 'AzureOpenAI' }
+  if (modelProvider === 'deepseek') return { modelProvider, client: 'DeepSeek' }
+  return { modelProvider, client: 'OpenAI' }
+}
+
 module.exports = {
   extractChatTemplateFromInstructions,
   normalizePromptVariables,
   extractTextFromContentItem,
   extractContentParts,
   hasMultimodalInputs,
+  getModelProviderAndClient,
   getOpenAIModelProvider,
 }

package/packages/dd-trace/src/llmobs/prompts/cache.js

@@ -0,0 +1,272 @@
+'use strict'
+
+const { createHash, randomUUID } = require('node:crypto')
+const fs = require('node:fs')
+const os = require('node:os')
+const path = require('node:path')
+
+const { LRUCache } = require('../../../../../vendor/dist/lru-cache')
+
+const log = require('../../log')
+const ManagedPrompt = require('./prompt')
+
+const CACHE_SUBPATH = path.join('datadog', 'llmobs', 'prompts')
+const MAX_HOT_ENTRIES = 1024
+const PROMPT_SOURCES = new Set(['registry', 'cache', 'fallback', 'ff', 'resolve'])
+
+/** @param {string} key */
+function promptIdFromKey (key) {
+  return key.slice(0, key.lastIndexOf(':'))
+}
+
+/**
+ * @param {string} promptId
+ * @param {Array<unknown>} selector
+ */
+function cacheKey (promptId, selector) {
+  const hash = createHash('sha1').update(JSON.stringify(selector)).digest('hex').slice(0, 16)
+  return `${promptId}:${hash}`
+}
+
+function defaultCacheDir () {
+  try {
+    const home = os.homedir()
+    if (home) return path.join(home, '.cache', CACHE_SUBPATH)
+  } catch {}
+  return path.join(os.tmpdir(), CACHE_SUBPATH)
+}
+
+class HotCache {
+  /**
+   * Create the bounded in-memory prompt cache.
+   * @param {object} options
+   * @param {number} options.ttlMs
+   * @param {(key: string, stale: ManagedPrompt | undefined, options: {context: object}) =>
+   *   Promise<ManagedPrompt | undefined>} [options.fetchMethod]
+   */
+  constructor ({ ttlMs, fetchMethod }) {
+    this.enabled = ttlMs > 0
+    this.cache = new LRUCache({
+      max: MAX_HOT_ENTRIES,
+      ttl: this.enabled && Number.isFinite(ttlMs) ? ttlMs : undefined,
+      allowStale: true,
+      noDeleteOnStaleGet: true,
+      fetchMethod,
+    })
+  }
+
+  /**
+   * Read a prompt and its freshness.
+   * @param {string} key
+   */
+  get (key) {
+    if (!this.enabled) return
+    const status = {}
+    const prompt = this.cache.get(key, { status })
+    if (!prompt) return
+    return { prompt, stale: status.get === 'stale' }
+  }
+
+  /**
+   * Store a prompt.
+   * @param {string} key
+   * @param {ManagedPrompt} prompt
+   * @param {number} [ageMs]
+   */
+  set (key, prompt, ageMs = 0) {
+    if (this.enabled) this.cache.set(key, prompt, { start: performance.now() - ageMs })
+  }
+
+  /**
+   * Refresh a stale prompt in the background.
+   * @param {string} key
+   * @param {object} context
+   */
+  refresh (key, context) {
+    if (!this.enabled) return
+    void this.cache.fetch(key, {
+      allowStaleOnFetchRejection: true,
+      context,
+      forceRefresh: true,
+      noDeleteOnFetchRejection: true,
+    }).catch(() => {})
+  }
+
+  /**
+   * Delete one selector.
+   * @param {string} key
+   */
+  delete (key) {
+    if (this.enabled) this.cache.delete(key)
+  }
+
+  /**
+   * Clear all hot prompt entries.
+   */
+  clear () {
+    if (this.enabled) this.cache.clear()
+  }
+
+  /**
+   * Evict every selector for one exact prompt ID.
+   * @param {string} promptId
+   */
+  evictPrompt (promptId) {
+    if (!this.enabled) return
+    for (const [key] of this.cache.dump()) {
+      if (promptIdFromKey(key) === promptId) this.cache.delete(key)
+    }
+  }
+}
+
+class WarmCache {
+  /**
+   * Create the optional filesystem prompt cache.
+   * @param {object} options
+   * @param {string | undefined} options.cacheDir
+   * @param {boolean} [options.enabled]
+   * @param {number} options.ttlMs
+   */
+  constructor ({ cacheDir, enabled = true, ttlMs }) {
+    this.enabled = enabled && ttlMs > 0
+    this.ttlMs = ttlMs
+    this.cacheDir = cacheDir || defaultCacheDir()
+    if (this.enabled) this.#ensureDir(this.cacheDir)
+  }
+
+  /**
+   * Create a secure cache directory.
+   * @param {string} directory
+   */
+  #ensureDir (directory) {
+    try {
+      fs.mkdirSync(directory, { recursive: true, mode: 0o700 })
+    } catch (error) {
+      log.warn('Failed to create prompt cache directory: %s', error.message)
+      this.enabled = false
+    }
+  }
+
+  /**
+   * Resolve the collision-safe directory for a prompt ID.
+   * @param {string} promptId
+   */
+  #promptDir (promptId) {
+    return path.join(this.cacheDir, Buffer.from(promptId).toString('base64url') || '_')
+  }
+
+  /**
+   * Resolve the cache file for a selector key.
+   * @param {string} key
+   */
+  #path (key) {
+    const index = key.lastIndexOf(':')
+    return path.join(this.#promptDir(key.slice(0, index)), `${key.slice(index + 1)}.json`)
+  }
+
+  /**
+   * Restore and validate a prompt from the warm cache.
+   * @param {object} data
+   */
+  #deserialize (data) {
+    const validTemplate = typeof data?.template === 'string' || (
+      Array.isArray(data?.template) && data.template.every(message => {
+        return message && typeof message.role === 'string' && typeof message.content === 'string'
+      })
+    )
+    if (
+      typeof data?.id !== 'string' ||
+      typeof data.version !== 'string' ||
+      !PROMPT_SOURCES.has(data.source) ||
+      !validTemplate ||
+      (data.promptUuid !== undefined && typeof data.promptUuid !== 'string') ||
+      (data.promptVersionUuid !== undefined && typeof data.promptVersionUuid !== 'string')
+    ) {
+      throw new TypeError('Invalid managed prompt cache entry')
+    }
+    return new ManagedPrompt(data)
+  }
+
+  /**
+   * Read a prompt and its freshness.
+   * @param {string} key
+   */
+  get (key) {
+    if (!this.enabled) return
+    try {
+      const data = JSON.parse(fs.readFileSync(this.#path(key), 'utf8'))
+      if (!Number.isFinite(data.timestamp)) throw new TypeError('Invalid prompt cache timestamp')
+      const prompt = this.#deserialize(data.prompt)
+      const ageMs = Math.max(0, Date.now() - data.timestamp)
+      return { prompt, stale: ageMs > this.ttlMs, ageMs }
+    } catch (error) {
+      log.debug('Failed to read prompt from cache: %s', error.message)
+    }
+  }
+
+  /**
+   * Store a prompt with restrictive permissions.
+   * @param {string} key
+   * @param {ManagedPrompt} prompt
+   */
+  set (key, prompt) {
+    if (!this.enabled) return
+    const file = this.#path(key)
+    const temporary = `${file}.tmp.${randomUUID()}`
+    try {
+      this.#ensureDir(path.dirname(file))
+      if (!this.enabled) return
+      // Write separately, then rename so other processes see either the old file or the complete new one.
+      fs.writeFileSync(temporary, JSON.stringify({ prompt, timestamp: Date.now() }), {
+        encoding: 'utf8',
+        mode: 0o600,
+      })
+      fs.renameSync(temporary, file)
+    } catch (error) {
+      try { fs.rmSync(temporary, { force: true }) } catch {}
+      log.debug('Failed to write prompt to cache: %s', error.message)
+    }
+  }
+
+  /**
+   * Delete one selector.
+   * @param {string} key
+   */
+  delete (key) {
+    if (!this.enabled) return
+    try {
+      fs.rmSync(this.#path(key), { force: true })
+    } catch (error) {
+      log.debug('Failed to delete prompt from cache: %s', error.message)
+    }
+  }
+
+  /**
+   * Evict every selector for one exact prompt ID.
+   * @param {string} promptId
+   */
+  evictPrompt (promptId) {
+    if (!this.enabled) return
+    try {
+      fs.rmSync(this.#promptDir(promptId), { recursive: true, force: true })
+    } catch (error) {
+      log.debug('Failed to evict prompt from cache: %s', error.message)
+    }
+  }
+
+  /**
+   * Clear all warm prompt entries.
+   */
+  clear () {
+    if (!this.enabled) return
+    try {
+      for (const entry of fs.readdirSync(this.cacheDir)) {
+        fs.rmSync(path.join(this.cacheDir, entry), { recursive: true, force: true })
+      }
+    } catch (error) {
+      log.debug('Failed to clear prompt cache: %s', error.message)
+    }
+  }
+}
+
+module.exports = { HotCache, WarmCache, cacheKey, promptIdFromKey }