dd-trace 5.123.0 → 5.124.0

package/packages/dd-trace/src/llmobs/experiments/index.js

@@ -2,11 +2,13 @@
 
 const log = require('../../log')
 const { ExperimentsClient } = require('./client')
-const { Dataset, DatasetRecord } = require('./dataset')
+const { Dataset } = require('./dataset')
 const { Experiment, ExternalExperiment } = require('./experiment')
 const { validateTagsList } = require('./util')
 const NoopExperiments = require('./noop')
 
+const DEFAULT_PROJECT_NAME = 'default-project'
+
 // Poll `attempt` with exponential backoff until it returns true or the time
 // budget is spent. Used for eventually-consistent reads (pullDataset).
 async function retryWithBackoff (attempt, { maxTotalMs = 30_000, baseDelayMs = 250, maxDelayMs = 8000 } = {}) {
@@ -33,8 +35,8 @@ class Experiments {
 
   constructor (config, llmobs) {
     this.#config = config
-    this.#llmobs = llmobs
-    this.#projectName = config.llmobs?.mlApp || config.service
+    this.#llmobs = config.llmobs?.mlApp || config.service ? llmobs : undefined
+    this.#projectName = config.llmobs?.projectName || DEFAULT_PROJECT_NAME
     this.#client = this.#clientForProject(this.#projectName)
   }
 
@@ -51,27 +53,26 @@ class Experiments {
     })
   }
 
+  /**
+   * @param {string | undefined} projectName
+   * @returns {ExperimentsClient}
+   */
+  #clientForOperation (projectName) {
+    if (projectName !== undefined && projectName !== this.#projectName) {
+      return this.#clientForProject(projectName)
+    }
+    if (this.#client === undefined) this.#client = this.#clientForProject(projectName)
+    return this.#client
+  }
+
   // Create a local dataset buffer. Pushed remotely on first experiment run.
   createDataset (name, descriptionOrOptions = '') {
     const options = typeof descriptionOrOptions === 'string'
       ? { description: descriptionOrOptions }
       : (descriptionOrOptions ?? {})
-    const dataset = new Dataset(this.#client, name, options.description ?? '')
-    const recordIds = new Set()
-    if ((options.records) != null) {
-      for (const record of options.records) {
-        if (record.id !== undefined && (typeof record.id !== 'string' || record.id.length === 0)) {
-          throw new Error('record id must be a non-empty string')
-        }
-        if (record.id !== undefined) {
-          if (recordIds.has(record.id)) throw new Error(`Duplicate record id '${record.id}'`)
-          recordIds.add(record.id)
-        }
-        dataset.addRecord(
-          new DatasetRecord(record.inputData, record.expectedOutput, record.metadata, record.id, record.tags)
-        )
-      }
-    }
+    const client = this.#clientForOperation(options.projectName)
+    const dataset = new Dataset(client, name, options.description ?? '')
+    if ((options.records) != null) dataset.addRecords(options.records)
     return dataset
   }
 
@@ -80,9 +81,11 @@ class Experiments {
   // wait until that many records are readable. Pass `tags` to filter records by
   // dataset record tags.
   async pullDataset (name, options = {}) {
-    const { expectedRecordCount, maxWaitMs = 30_000, tags, version } = options
+    const { expectedRecordCount, maxWaitMs = 30_000, projectName, tags, version } = options
     const filterTags = validateTagsList(tags)
-    const projectId = await this.#client.ensureProjectId()
+    const client = this.#clientForOperation(projectName)
+    const resolvedProjectName = projectName ?? this.#projectName
+    const projectId = await client.ensureProjectId()
 
     let pulledDataset = null
     let records = []
@@ -93,7 +96,7 @@ class Experiments {
     const succeeded = await retryWithBackoff(async () => {
       try {
         if (pulledDataset === null) {
-          const datasets = await this.#client.listDatasets(projectId, { name })
+          const datasets = await client.listDatasets(projectId, { name })
           for (const dataset of datasets) {
             if (dataset.name() === name) {
               pulledDataset = dataset
@@ -109,7 +112,7 @@ class Experiments {
         // Follow the meta.after / page[cursor] pagination until the last page.
         for (;;) {
           // eslint-disable-next-line no-await-in-loop
-          const page = await this.#client.listDatasetRecords(projectId, pulledDataset.id(), {
+          const page = await client.listDatasetRecords(projectId, pulledDataset.id(), {
             cursor,
             tags: filterTags,
             version: datasetVersion,
@@ -129,13 +132,13 @@ class Experiments {
     }, { maxTotalMs: maxWaitMs })
 
     if (pulledDataset === null && lastError) {
-      throw new Error(`Failed to list datasets in project '${this.#projectName}': ${lastError}`)
+      throw new Error(`Failed to list datasets in project '${resolvedProjectName}': ${lastError}`)
     }
     if (pulledDataset === null) {
-      throw new Error(`Dataset '${name}' not found in project '${this.#projectName}' (after ${maxWaitMs}ms)`)
+      throw new Error(`Dataset '${name}' not found in project '${resolvedProjectName}' (after ${maxWaitMs}ms)`)
     }
     if (!succeeded && lastError) {
-      throw new Error(`Failed to fetch records for dataset '${name}' in project '${this.#projectName}': ${lastError}`)
+      throw new Error(`Failed to fetch records for dataset '${name}' in project '${resolvedProjectName}': ${lastError}`)
     }
     if (!succeeded && expectedRecordCount != null) {
       throw new Error(
@@ -151,7 +154,7 @@ class Experiments {
     }
 
     return Dataset.fromExisting(
-      this.#client,
+      client,
       name,
       pulledDataset.description(),
       pulledDataset.id(),
@@ -163,9 +166,26 @@ class Experiments {
     )
   }
 
-  // Build an experiment: { name, dataset, task, evaluators, description?, config?, tags? }.
+  // Build an experiment with a dataset, task, evaluators, and optional project/config/tags.
   experiment (options) {
-    return new Experiment(this.#client, options, this.#llmobs)
+    const datasetProjectName = options?.dataset?.projectName?.()
+    if (options?.projectName !== undefined &&
+        datasetProjectName !== undefined &&
+        options.projectName !== datasetProjectName) {
+      throw new Error(
+        `Experiment project '${options.projectName}' does not match dataset project '${datasetProjectName}'`
+      )
+    }
+    const projectName = options?.projectName ?? datasetProjectName
+    const client = this.#clientForOperation(projectName)
+    const usesDatasetOverride = datasetProjectName !== undefined && datasetProjectName !== this.#projectName
+    const resolvedProjectName = projectName ?? this.#config.llmobs?.projectName
+    const experimentOptions = options?.projectName === undefined &&
+      (usesDatasetOverride || this.#config.llmobs?.projectName !== undefined) &&
+      resolvedProjectName !== undefined
+      ? { ...options, projectName: resolvedProjectName }
+      : options
+    return new Experiment(client, experimentOptions, this.#llmobs)
   }
 
   /**
@@ -177,10 +197,11 @@ class Experiments {
    * @returns {Promise<ExternalExperiment>}
    */
   startExperiment (options) {
-    const client = options?.projectName === undefined || options.projectName === this.#projectName
-      ? this.#client
-      : this.#clientForProject(options.projectName)
-    return new Experiment(client, { ...options, external: true }).start()
+    const client = this.#clientForOperation(options?.projectName)
+    const experimentOptions = options?.projectName === undefined && this.#config.llmobs?.projectName !== undefined
+      ? { ...options, projectName: this.#config.llmobs.projectName }
+      : options
+    return new Experiment(client, { ...experimentOptions, external: true }).start()
       .then(experiment => new ExternalExperiment(experiment))
   }
 }
@@ -195,15 +216,6 @@ function createExperiments (config, llmobs) {
     log.warn('LLMObs experiments: missing api and/or app keys, set DD_API_KEY and DD_APP_KEY')
     return new NoopExperiments('DD_API_KEY and DD_APP_KEY are required for experiments')
   }
-  if (!config.llmobs?.mlApp && !config.service) {
-    const reason = 'no project name configured; set the DD_LLMOBS_ML_APP environment variable (or llmobs.mlApp in ' +
-      'tracer.init()) to name the LLM Obs project, or DD_SERVICE (or service in tracer.init()) as a fallback, ' +
-      'then retry'
-    const experiments = new Experiments(config, llmobs)
-    return new NoopExperiments(reason, {
-      startExperiment: (options) => experiments.startExperiment(options),
-    })
-  }
   return new Experiments(config, llmobs)
 }
 

package/packages/dd-trace/src/llmobs/experiments/noop.js

@@ -7,6 +7,15 @@ const NOOP_EXPERIMENT_ID = '00000000-0000-0000-0000-000000000000'
 const NOOP_SPAN_ID = '0000000000000000'
 const NOOP_TRACE_ID = '00000000000000000000000000000000'
 
+/**
+ * @typedef {object} DatasetRecordNew
+ * @property {string} [id]
+ * @property {unknown} inputData
+ * @property {unknown} [expectedOutput]
+ * @property {Record<string, unknown>} [metadata]
+ * @property {string[]} [tags]
+ */
+
 class NoopDataset {
   #name
   #description
@@ -37,6 +46,24 @@ class NoopDataset {
     return this
   }
 
+  /**
+   * Add multiple records to a dataset.
+   * @param {DatasetRecordNew[]} records
+   * @returns {NoopDataset} This dataset for chaining.
+   */
+  addRecords (records) {
+    for (const record of records) {
+      this.#records.push({
+        id: record.id ?? null,
+        input: record.inputData,
+        expectedOutput: record.expectedOutput ?? null,
+        metadata: record.metadata ?? {},
+        tags: [...(record.tags ?? [])],
+      })
+    }
+    return this
+  }
+
   update (index, fields) {
     const record = this.#records[index]
     if (record == null) return this
@@ -114,6 +141,10 @@ class NoopDataset {
     return null
   }
 
+  projectName () {
+    return null
+  }
+
   version () {
     return null
   }
@@ -165,7 +196,13 @@ class NoopExperiment {
   }
 
   run () {
-    return Promise.resolve({ experimentId: null, rows: [], url: null })
+    return Promise.resolve({
+      experimentId: null,
+      rows: [],
+      summaryEvaluations: {},
+      runs: [],
+      url: null,
+    })
   }
 
   /**
@@ -200,11 +237,9 @@ class NoopExperiment {
 // throwing, so intentionally disabled experiments remain graceful.
 class NoopExperiments {
   #reason
-  #startExperiment
 
-  constructor (reason, options = {}) {
+  constructor (reason) {
     this.#reason = reason || 'LLMObs experiments are not available'
-    this.#startExperiment = options.startExperiment
   }
 
   #warn () {
@@ -231,10 +266,6 @@ class NoopExperiments {
    * @returns {Promise<ExternalExperiment>}
    */
   startExperiment (options = {}) {
-    if (this.#startExperiment !== undefined && options.projectName) {
-      return this.#startExperiment(options)
-    }
-
     this.#warn()
     return Promise.resolve(new ExternalExperiment(new NoopExperiment(options.name, true)))
   }

package/packages/dd-trace/src/llmobs/experiments/result.js

@@ -27,6 +27,7 @@ class ExperimentRun {
   constructor (fields) {
     this.runId = fields.runId
     this.runIteration = fields.runIteration
+    this.hasError = fields.hasError
     this.rows = fields.rows
     this.summaryEvaluations = fields.summaryEvaluations
   }

package/packages/dd-trace/src/llmobs/experiments/util.js

@@ -1,5 +1,7 @@
 'use strict'
 
+const { randomUUID } = require('node:crypto')
+
 const log = require('../../log')
 
 // Matches the backend and dd-trace-py evaluator metric label contract.
@@ -42,6 +44,23 @@ function tagOperationsAreEmpty (operations) {
   )
 }
 
+/**
+ * @param {unknown} value
+ * @param {string} name
+ * @returns {number}
+ */
+function normalizePositiveInteger (value, name) {
+  if (!Number.isInteger(value) || value < 1) throw new Error(`${name} must be a positive integer`)
+  return value
+}
+
+/**
+ * @returns {string}
+ */
+function generateRunId () {
+  return randomUUID()
+}
+
 /**
  * @param {string} name
  */
@@ -297,11 +316,13 @@ module.exports = {
   buildSpanMetadata,
   buildTags,
   durationNs,
+  generateRunId,
   hasEntries,
   inferMetricType,
   mergeTags,
   normalizeEvaluators,
   normalizeJsonMetricValue,
+  normalizePositiveInteger,
   recordTagsToObject,
   sleep,
   stringify,

package/packages/dd-trace/src/llmobs/index.js

@@ -3,7 +3,9 @@
 const { channel } = require('dc-polyfill')
 
 const { readDatadogTags, writeDatadogTags } = require('../carrier')
+const { registerTelemetryFlusher } = require('../flush')
 const log = require('../log')
+const { createServerlessDeliveryTracker } = require('../serverless')
 const { DD_MAJOR } = require('../../../../version')
 const startupLogs = require('../startup-log')
 const {
@@ -57,6 +59,8 @@ let spanWriter
 /** @type {LLMObsEvalMetricsWriter | null} */
 let evalWriter
 
+let unregisterTelemetryFlusher
+
 /** @type {import('../config/config-base')} */
 let globalTracerConfig
 
@@ -66,23 +70,36 @@ let globalTracerConfig
 function enable (config) {
   globalTracerConfig = config
 
+  const retiredSpanWriter = spanWriter
+  const retiredEvalWriter = evalWriter
+  const isReinitializing = Boolean(retiredSpanWriter || retiredEvalWriter)
+  unregisterTelemetryFlusher?.()
+  retireWriters(retiredSpanWriter, retiredEvalWriter)
+
   const startTime = performance.now()
   // create writers and eval writer append and flush channels
   // span writer append is handled by the span processor
   evalWriter = new LLMObsEvalMetricsWriter(config)
   spanWriter = new LLMObsSpanWriter(config)
+  const currentEvalWriter = evalWriter
+  const currentSpanWriter = spanWriter
+  unregisterTelemetryFlusher = registerTelemetryFlusher(done => {
+    flushWriters(done, currentSpanWriter, currentEvalWriter)
+  })
 
-  evalMetricAppendCh.subscribe(handleEvalMetricAppend)
-  flushCh.subscribe(handleFlush)
-  registerUserSpanProcessorCh.subscribe(handleRegisterProcessor)
+  if (!isReinitializing) {
+    evalMetricAppendCh.subscribe(handleEvalMetricAppend)
+    flushCh.subscribe(handleFlush)
+    registerUserSpanProcessorCh.subscribe(handleRegisterProcessor)
+  }
 
   // span processing
   spanProcessor = new LLMObsSpanProcessor(config)
   spanProcessor.setWriter(spanWriter)
-  spanFinishCh.subscribe(handleSpanProcess)
+  if (!isReinitializing) spanFinishCh.subscribe(handleSpanProcess)
 
   // distributed tracing for llmobs
-  injectCh.subscribe(handleLLMObsInjection)
+  if (!isReinitializing) injectCh.subscribe(handleLLMObsInjection)
 
   setAgentStrategy(config, useAgentless => {
     if (useAgentless && !(config.DD_API_KEY && config.site)) {
@@ -94,8 +111,10 @@ function enable (config) {
       }
     }
 
-    evalWriter?.setAgentless(useAgentless)
-    spanWriter?.setAgentless(useAgentless)
+    // A disable can happen while transport selection is still pending. Keep
+    // configuring these writers so their queued lifecycle flushes can drain.
+    currentEvalWriter.setAgentless(useAgentless)
+    currentSpanWriter.setAgentless(useAgentless)
 
     telemetry.recordLLMObsEnabled(startTime, config)
     log.debug('[LLMObs] Enabled LLM Observability with configuration: %o', config.llmobs)
@@ -109,16 +128,39 @@ function disable () {
   if (injectCh.hasSubscribers) injectCh.unsubscribe(handleLLMObsInjection)
   if (registerUserSpanProcessorCh.hasSubscribers) registerUserSpanProcessorCh.unsubscribe(handleRegisterProcessor)
 
-  spanWriter?.destroy()
-  evalWriter?.destroy()
+  const retiredSpanWriter = spanWriter
+  const retiredEvalWriter = evalWriter
   spanProcessor?.setWriter(null)
+  unregisterTelemetryFlusher?.()
+  unregisterTelemetryFlusher = undefined
 
   spanWriter = null
   evalWriter = null
 
+  retireWriters(retiredSpanWriter, retiredEvalWriter)
+
   log.debug('[LLMObs] Disabled LLM Observability')
 }
 
+/**
+ * Keeps retired writers reachable until their destroy-triggered deliveries complete.
+ * @param {LLMObsSpanWriter | null} retiredSpanWriter
+ * @param {LLMObsEvalMetricsWriter | null} retiredEvalWriter
+ * @returns {void}
+ */
+function retireWriters (retiredSpanWriter, retiredEvalWriter) {
+  const retiredWriters = [retiredSpanWriter, retiredEvalWriter].filter(Boolean)
+  if (retiredWriters.length === 0) return
+  let remainingWriters = retiredWriters.length
+  const unregisterRetiredFlusher = registerTelemetryFlusher(done => {
+    flushWriters(done, retiredSpanWriter, retiredEvalWriter)
+  })
+  function onWriterDestroyed () {
+    if (--remainingWriters === 0) unregisterRetiredFlusher?.()
+  }
+  for (const writer of retiredWriters) writer.destroy(onWriterDestroyed)
+}
+
 // since LLMObs traces can extend between services and be the same trace,
 // we need to propagate the parent id, mlApp, session id, and sampling rate/decision.
 function handleLLMObsInjection ({ carrier }) {
@@ -192,15 +234,36 @@ function handleLLMObsInjection ({ carrier }) {
   if (tags !== existing) writeDatadogTags(carrier, tags)
 }
 
-function handleFlush () {
-  let err = ''
-  try {
-    spanWriter.flush()
-    evalWriter.flush()
-  } catch (e) {
-    err = 'writer_flush_error'
-    log.warn('Failed to flush LLMObs spans and evaluation metrics:', e.message)
+/**
+ * Flushes the specified LLMObs writers and joins deliveries active at the boundary.
+ * @param {Function} [done]
+ * @param {LLMObsSpanWriter | null} [currentSpanWriter]
+ * @param {LLMObsEvalMetricsWriter | null} [currentEvalWriter]
+ * @returns {boolean} `true` when a writer throws synchronously.
+ */
+function flushWriters (done, currentSpanWriter = spanWriter, currentEvalWriter = evalWriter) {
+  let failed = false
+  const deliveryTracker = createServerlessDeliveryTracker()
+  const flush = writer => {
+    try {
+      if (deliveryTracker && writer) deliveryTracker.track(complete => writer.flush(complete))
+      // Non-serverless flushes retain the existing writer behavior.
+      else writer?.flush()
+    } catch (error) {
+      failed = true
+      log.warn('Failed to flush LLMObs writer:', error.message)
+    }
   }
+
+  flush(currentSpanWriter)
+  flush(currentEvalWriter)
+  deliveryTracker?.waitForIdle(done)
+  if (!deliveryTracker) done?.()
+  return failed
+}
+
+function handleFlush () {
+  const err = flushWriters() ? 'writer_flush_error' : ''
   telemetry.recordUserFlush(err)
 }
 

package/packages/dd-trace/src/llmobs/noop.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const NoopExperiments = require('./experiments/noop')
+let NoopExperiments
 
 class NoopLLMObs {
   constructor (noopTracer) {
@@ -12,6 +12,7 @@ class NoopLLMObs {
   }
 
   get experiments () {
+    NoopExperiments ??= require('./experiments/noop')
     return new NoopExperiments('LLM Observability is not enabled')
   }
 

package/packages/dd-trace/src/llmobs/plugins/anthropic/util.js

@@ -38,16 +38,20 @@ function formatAnthropicToolResultContent (content) {
   if (typeof content === 'string') {
     return content
   } else if (Array.isArray(content)) {
-    const formattedContent = []
+    let formattedContent = ''
     for (const toolResultBlock of content) {
+      let part
       if (toolResultBlock.text) {
-        formattedContent.push(toolResultBlock.text)
+        part = toolResultBlock.text
       } else if (toolResultBlock.type === 'image') {
-        formattedContent.push('([IMAGE DETECTED])')
+        part = '([IMAGE DETECTED])'
       }
+      if (part === undefined) continue
+      if (formattedContent) formattedContent += ','
+      formattedContent += part
     }
 
-    return formattedContent.join(',')
+    return formattedContent
   }
   return JSON.stringify(content)
 }

package/packages/dd-trace/src/llmobs/plugins/claude-agent-sdk/index.js

@@ -18,12 +18,15 @@ function getToolOutputText (raw) {
   if (raw == null) return
 
   if (Array.isArray(raw)) {
-    const output = []
+    let output = ''
+    let separator = ''
     for (const block of raw) {
       const text = getToolOutputText(block)
-      if (text) output.push(text)
+      if (!text) continue
+      output += separator + text
+      separator = '\n'
     }
-    return output.join('\n') || undefined
+    return output
   }
 
   if (raw.type === 'tool_result') return getToolOutputText(raw.content)

package/packages/dd-trace/src/llmobs/plugins/openai/utils.js

@@ -134,15 +134,17 @@ function hasMultimodalInputs (variables) {
  * @returns {{ content: string, audioParts: Array<{ mimeType: string, content: string }> }}
  */
 function extractContentParts (parts) {
-  const extracted = []
+  let content = ''
+  let hasContent = false
   const audioParts = []
 
   for (const part of parts) {
     const partType = part?.type ?? ''
+    let extracted
     if (partType === 'text') {
-      extracted.push(part.text ?? '')
+      extracted = part.text ?? ''
     } else if (partType === 'image_url') {
-      extracted.push(IMAGE_FALLBACK)
+      extracted = IMAGE_FALLBACK
     } else if (partType === 'input_audio') {
       const inputAudio = part.input_audio ?? {}
       const data = inputAudio.data
@@ -151,14 +153,19 @@ function extractContentParts (parts) {
         // is needed. Only fall back to "[audio]" when there's no audio to capture.
         audioParts.push(formatAudioPart(data, audioMimeTypeFromFormat(inputAudio.format, AUDIO_MIME_TYPES)))
       } else {
-        extracted.push(AUDIO_FALLBACK)
+        extracted = AUDIO_FALLBACK
       }
     } else {
-      extracted.push(`[${partType}]`)
+      extracted = `[${partType}]`
     }
+
+    if (extracted === undefined) continue
+    if (hasContent) content += '\n'
+    content += extracted
+    hasContent = true
   }
 
-  return { content: extracted.join('\n'), audioParts }
+  return { content, audioParts }
 }
 
 /**

package/packages/dd-trace/src/llmobs/plugins/openai-agents/utils.js

@@ -11,23 +11,23 @@ const { safeJsonParse } = require('../../util')
  * @returns {{ content: string, audioParts: Array<{ mimeType: string, content: string }> }}
  */
 function extractMessageContent (parts) {
-  const contentParts = []
+  let content = ''
   const audioParts = []
 
   for (const part of parts) {
     if (!part) continue
     const text = extractTextFromContentItem(part)
     if (text) {
-      contentParts.push(text)
+      content += text
       continue
     }
 
     const extracted = extractContentParts([part])
-    if (extracted.content) contentParts.push(extracted.content)
+    if (extracted.content) content += extracted.content
     if (extracted.audioParts.length > 0) audioParts.push(...extracted.audioParts)
   }
 
-  return { content: contentParts.join(''), audioParts }
+  return { content, audioParts }
 }
 
 /**
@@ -167,13 +167,11 @@ function extractOutputMessages (result) {
       if (item.type === 'message') {
         let content = ''
         if (Array.isArray(item.content)) {
-          const textParts = []
           for (const contentItem of item.content) {
             if (contentItem?.type === 'output_text' && contentItem.text) {
-              textParts.push(contentItem.text)
+              content += contentItem.text
             }
           }
-          content = textParts.join('')
         } else if (typeof item.content === 'string') {
           content = item.content
         }