dd-trace 6.12.0 → 6.14.0

package/packages/dd-trace/src/llmobs/experiments/index.js

@@ -2,11 +2,13 @@
 
 const log = require('../../log')
 const { ExperimentsClient } = require('./client')
-const { Dataset, DatasetRecord } = require('./dataset')
+const { Dataset } = require('./dataset')
 const { Experiment, ExternalExperiment } = require('./experiment')
 const { validateTagsList } = require('./util')
 const NoopExperiments = require('./noop')
 
+const DEFAULT_PROJECT_NAME = 'default-project'
+
 // Poll `attempt` with exponential backoff until it returns true or the time
 // budget is spent. Used for eventually-consistent reads (pullDataset).
 async function retryWithBackoff (attempt, { maxTotalMs = 30_000, baseDelayMs = 250, maxDelayMs = 8000 } = {}) {
@@ -33,8 +35,8 @@ class Experiments {
 
   constructor (config, llmobs) {
     this.#config = config
-    this.#llmobs = llmobs
-    this.#projectName = config.llmobs?.mlApp || config.service
+    this.#llmobs = config.llmobs?.mlApp || config.service ? llmobs : undefined
+    this.#projectName = config.llmobs?.projectName || DEFAULT_PROJECT_NAME
     this.#client = this.#clientForProject(this.#projectName)
   }
 
@@ -51,27 +53,26 @@ class Experiments {
     })
   }
 
+  /**
+   * @param {string | undefined} projectName
+   * @returns {ExperimentsClient}
+   */
+  #clientForOperation (projectName) {
+    if (projectName !== undefined && projectName !== this.#projectName) {
+      return this.#clientForProject(projectName)
+    }
+    if (this.#client === undefined) this.#client = this.#clientForProject(projectName)
+    return this.#client
+  }
+
   // Create a local dataset buffer. Pushed remotely on first experiment run.
   createDataset (name, descriptionOrOptions = '') {
     const options = typeof descriptionOrOptions === 'string'
       ? { description: descriptionOrOptions }
       : (descriptionOrOptions ?? {})
-    const dataset = new Dataset(this.#client, name, options.description ?? '')
-    const recordIds = new Set()
-    if ((options.records) != null) {
-      for (const record of options.records) {
-        if (record.id !== undefined && (typeof record.id !== 'string' || record.id.length === 0)) {
-          throw new Error('record id must be a non-empty string')
-        }
-        if (record.id !== undefined) {
-          if (recordIds.has(record.id)) throw new Error(`Duplicate record id '${record.id}'`)
-          recordIds.add(record.id)
-        }
-        dataset.addRecord(
-          new DatasetRecord(record.inputData, record.expectedOutput, record.metadata, record.id, record.tags)
-        )
-      }
-    }
+    const client = this.#clientForOperation(options.projectName)
+    const dataset = new Dataset(client, name, options.description ?? '')
+    if ((options.records) != null) dataset.addRecords(options.records)
     return dataset
   }
 
@@ -80,9 +81,11 @@ class Experiments {
   // wait until that many records are readable. Pass `tags` to filter records by
   // dataset record tags.
   async pullDataset (name, options = {}) {
-    const { expectedRecordCount, maxWaitMs = 30_000, tags, version } = options
+    const { expectedRecordCount, maxWaitMs = 30_000, projectName, tags, version } = options
     const filterTags = validateTagsList(tags)
-    const projectId = await this.#client.ensureProjectId()
+    const client = this.#clientForOperation(projectName)
+    const resolvedProjectName = projectName ?? this.#projectName
+    const projectId = await client.ensureProjectId()
 
     let pulledDataset = null
     let records = []
@@ -93,7 +96,7 @@ class Experiments {
     const succeeded = await retryWithBackoff(async () => {
       try {
         if (pulledDataset === null) {
-          const datasets = await this.#client.listDatasets(projectId, { name })
+          const datasets = await client.listDatasets(projectId, { name })
           for (const dataset of datasets) {
             if (dataset.name() === name) {
               pulledDataset = dataset
@@ -109,7 +112,7 @@ class Experiments {
         // Follow the meta.after / page[cursor] pagination until the last page.
         for (;;) {
           // eslint-disable-next-line no-await-in-loop
-          const page = await this.#client.listDatasetRecords(projectId, pulledDataset.id(), {
+          const page = await client.listDatasetRecords(projectId, pulledDataset.id(), {
             cursor,
             tags: filterTags,
             version: datasetVersion,
@@ -129,13 +132,13 @@ class Experiments {
     }, { maxTotalMs: maxWaitMs })
 
     if (pulledDataset === null && lastError) {
-      throw new Error(`Failed to list datasets in project '${this.#projectName}': ${lastError}`)
+      throw new Error(`Failed to list datasets in project '${resolvedProjectName}': ${lastError}`)
     }
     if (pulledDataset === null) {
-      throw new Error(`Dataset '${name}' not found in project '${this.#projectName}' (after ${maxWaitMs}ms)`)
+      throw new Error(`Dataset '${name}' not found in project '${resolvedProjectName}' (after ${maxWaitMs}ms)`)
     }
     if (!succeeded && lastError) {
-      throw new Error(`Failed to fetch records for dataset '${name}' in project '${this.#projectName}': ${lastError}`)
+      throw new Error(`Failed to fetch records for dataset '${name}' in project '${resolvedProjectName}': ${lastError}`)
     }
     if (!succeeded && expectedRecordCount != null) {
       throw new Error(
@@ -151,7 +154,7 @@ class Experiments {
     }
 
     return Dataset.fromExisting(
-      this.#client,
+      client,
       name,
       pulledDataset.description(),
       pulledDataset.id(),
@@ -163,9 +166,26 @@ class Experiments {
     )
   }
 
-  // Build an experiment: { name, dataset, task, evaluators, description?, config?, tags? }.
+  // Build an experiment with a dataset, task, evaluators, and optional project/config/tags.
   experiment (options) {
-    return new Experiment(this.#client, options, this.#llmobs)
+    const datasetProjectName = options?.dataset?.projectName?.()
+    if (options?.projectName !== undefined &&
+        datasetProjectName !== undefined &&
+        options.projectName !== datasetProjectName) {
+      throw new Error(
+        `Experiment project '${options.projectName}' does not match dataset project '${datasetProjectName}'`
+      )
+    }
+    const projectName = options?.projectName ?? datasetProjectName
+    const client = this.#clientForOperation(projectName)
+    const usesDatasetOverride = datasetProjectName !== undefined && datasetProjectName !== this.#projectName
+    const resolvedProjectName = projectName ?? this.#config.llmobs?.projectName
+    const experimentOptions = options?.projectName === undefined &&
+      (usesDatasetOverride || this.#config.llmobs?.projectName !== undefined) &&
+      resolvedProjectName !== undefined
+      ? { ...options, projectName: resolvedProjectName }
+      : options
+    return new Experiment(client, experimentOptions, this.#llmobs)
   }
 
   /**
@@ -177,10 +197,11 @@ class Experiments {
    * @returns {Promise<ExternalExperiment>}
    */
   startExperiment (options) {
-    const client = options?.projectName === undefined || options.projectName === this.#projectName
-      ? this.#client
-      : this.#clientForProject(options.projectName)
-    return new Experiment(client, { ...options, external: true }).start()
+    const client = this.#clientForOperation(options?.projectName)
+    const experimentOptions = options?.projectName === undefined && this.#config.llmobs?.projectName !== undefined
+      ? { ...options, projectName: this.#config.llmobs.projectName }
+      : options
+    return new Experiment(client, { ...experimentOptions, external: true }).start()
       .then(experiment => new ExternalExperiment(experiment))
   }
 }
@@ -195,15 +216,6 @@ function createExperiments (config, llmobs) {
     log.warn('LLMObs experiments: missing api and/or app keys, set DD_API_KEY and DD_APP_KEY')
     return new NoopExperiments('DD_API_KEY and DD_APP_KEY are required for experiments')
   }
-  if (!config.llmobs?.mlApp && !config.service) {
-    const reason = 'no project name configured; set the DD_LLMOBS_ML_APP environment variable (or llmobs.mlApp in ' +
-      'tracer.init()) to name the LLM Obs project, or DD_SERVICE (or service in tracer.init()) as a fallback, ' +
-      'then retry'
-    const experiments = new Experiments(config, llmobs)
-    return new NoopExperiments(reason, {
-      startExperiment: (options) => experiments.startExperiment(options),
-    })
-  }
   return new Experiments(config, llmobs)
 }
 

package/packages/dd-trace/src/llmobs/experiments/noop.js

@@ -7,6 +7,15 @@ const NOOP_EXPERIMENT_ID = '00000000-0000-0000-0000-000000000000'
 const NOOP_SPAN_ID = '0000000000000000'
 const NOOP_TRACE_ID = '00000000000000000000000000000000'
 
+/**
+ * @typedef {object} DatasetRecordNew
+ * @property {string} [id]
+ * @property {unknown} inputData
+ * @property {unknown} [expectedOutput]
+ * @property {Record<string, unknown>} [metadata]
+ * @property {string[]} [tags]
+ */
+
 class NoopDataset {
   #name
   #description
@@ -37,6 +46,24 @@ class NoopDataset {
     return this
   }
 
+  /**
+   * Add multiple records to a dataset.
+   * @param {DatasetRecordNew[]} records
+   * @returns {NoopDataset} This dataset for chaining.
+   */
+  addRecords (records) {
+    for (const record of records) {
+      this.#records.push({
+        id: record.id ?? null,
+        input: record.inputData,
+        expectedOutput: record.expectedOutput ?? null,
+        metadata: record.metadata ?? {},
+        tags: [...(record.tags ?? [])],
+      })
+    }
+    return this
+  }
+
   update (index, fields) {
     const record = this.#records[index]
     if (record == null) return this
@@ -114,6 +141,10 @@ class NoopDataset {
     return null
   }
 
+  projectName () {
+    return null
+  }
+
   version () {
     return null
   }
@@ -165,7 +196,13 @@ class NoopExperiment {
   }
 
   run () {
-    return Promise.resolve({ experimentId: null, rows: [], url: null })
+    return Promise.resolve({
+      experimentId: null,
+      rows: [],
+      summaryEvaluations: {},
+      runs: [],
+      url: null,
+    })
   }
 
   /**
@@ -200,11 +237,9 @@ class NoopExperiment {
 // throwing, so intentionally disabled experiments remain graceful.
 class NoopExperiments {
   #reason
-  #startExperiment
 
-  constructor (reason, options = {}) {
+  constructor (reason) {
     this.#reason = reason || 'LLMObs experiments are not available'
-    this.#startExperiment = options.startExperiment
   }
 
   #warn () {
@@ -231,10 +266,6 @@ class NoopExperiments {
    * @returns {Promise<ExternalExperiment>}
    */
   startExperiment (options = {}) {
-    if (this.#startExperiment !== undefined && options.projectName) {
-      return this.#startExperiment(options)
-    }
-
     this.#warn()
     return Promise.resolve(new ExternalExperiment(new NoopExperiment(options.name, true)))
   }

package/packages/dd-trace/src/llmobs/experiments/result.js

@@ -27,6 +27,7 @@ class ExperimentRun {
   constructor (fields) {
     this.runId = fields.runId
     this.runIteration = fields.runIteration
+    this.hasError = fields.hasError
     this.rows = fields.rows
     this.summaryEvaluations = fields.summaryEvaluations
   }

package/packages/dd-trace/src/llmobs/experiments/util.js

@@ -1,5 +1,7 @@
 'use strict'
 
+const { randomUUID } = require('node:crypto')
+
 const log = require('../../log')
 
 // Matches the backend and dd-trace-py evaluator metric label contract.
@@ -42,6 +44,23 @@ function tagOperationsAreEmpty (operations) {
   )
 }
 
+/**
+ * @param {unknown} value
+ * @param {string} name
+ * @returns {number}
+ */
+function normalizePositiveInteger (value, name) {
+  if (!Number.isInteger(value) || value < 1) throw new Error(`${name} must be a positive integer`)
+  return value
+}
+
+/**
+ * @returns {string}
+ */
+function generateRunId () {
+  return randomUUID()
+}
+
 /**
  * @param {string} name
  */
@@ -297,11 +316,13 @@ module.exports = {
   buildSpanMetadata,
   buildTags,
   durationNs,
+  generateRunId,
   hasEntries,
   inferMetricType,
   mergeTags,
   normalizeEvaluators,
   normalizeJsonMetricValue,
+  normalizePositiveInteger,
   recordTagsToObject,
   sleep,
   stringify,

package/packages/dd-trace/src/llmobs/index.js

@@ -2,8 +2,9 @@
 
 const { channel } = require('dc-polyfill')
 
-const { readDatadogTags, writeDatadogTags } = require('../carrier')
+const { registerTelemetryFlusher } = require('../flush')
 const log = require('../log')
+const { createServerlessDeliveryTracker } = require('../serverless')
 const { DD_MAJOR } = require('../../../../version')
 const startupLogs = require('../startup-log')
 const {
@@ -23,7 +24,7 @@ const {
   PROPAGATED_TRACE_ID_KEY,
 } = require('./constants/tags')
 const { storage } = require('./storage')
-const { agentNameWireSafe, appendOptionalPropagatedTag, resolveAgentAttribution, stripTagsetEntry } = require('./util')
+const { agentNameWireSafe, resolveAgentAttribution } = require('./util')
 const telemetry = require('./telemetry')
 const LLMObsSpanProcessor = require('./span_processor')
 const LLMObsEvalMetricsWriter = require('./writers/evaluations')
@@ -57,32 +58,54 @@ let spanWriter
 /** @type {LLMObsEvalMetricsWriter | null} */
 let evalWriter
 
+let unregisterTelemetryFlusher
+
 /** @type {import('../config/config-base')} */
 let globalTracerConfig
 
+/**
+ * @typedef {object} TraceTagInjection
+ * @property {import('../opentracing/span_context')} spanContext
+ * @property {Array<string | undefined>} [traceTagReplacements]
+ * @property {number} [optionalTraceTagCount]
+ */
+
 /**
  * @param {@type import('../config/config-base')} config
  */
 function enable (config) {
   globalTracerConfig = config
 
+  const retiredSpanWriter = spanWriter
+  const retiredEvalWriter = evalWriter
+  const isReinitializing = Boolean(retiredSpanWriter || retiredEvalWriter)
+  unregisterTelemetryFlusher?.()
+  retireWriters(retiredSpanWriter, retiredEvalWriter)
+
   const startTime = performance.now()
   // create writers and eval writer append and flush channels
   // span writer append is handled by the span processor
   evalWriter = new LLMObsEvalMetricsWriter(config)
   spanWriter = new LLMObsSpanWriter(config)
+  const currentEvalWriter = evalWriter
+  const currentSpanWriter = spanWriter
+  unregisterTelemetryFlusher = registerTelemetryFlusher(done => {
+    flushWriters(done, currentSpanWriter, currentEvalWriter)
+  })
 
-  evalMetricAppendCh.subscribe(handleEvalMetricAppend)
-  flushCh.subscribe(handleFlush)
-  registerUserSpanProcessorCh.subscribe(handleRegisterProcessor)
+  if (!isReinitializing) {
+    evalMetricAppendCh.subscribe(handleEvalMetricAppend)
+    flushCh.subscribe(handleFlush)
+    registerUserSpanProcessorCh.subscribe(handleRegisterProcessor)
+  }
 
   // span processing
   spanProcessor = new LLMObsSpanProcessor(config)
   spanProcessor.setWriter(spanWriter)
-  spanFinishCh.subscribe(handleSpanProcess)
+  if (!isReinitializing) spanFinishCh.subscribe(handleSpanProcess)
 
   // distributed tracing for llmobs
-  injectCh.subscribe(handleLLMObsInjection)
+  if (!isReinitializing) injectCh.subscribe(handleLLMObsInjection)
 
   setAgentStrategy(config, useAgentless => {
     if (useAgentless && !(config.DD_API_KEY && config.site)) {
@@ -94,8 +117,10 @@ function enable (config) {
       }
     }
 
-    evalWriter?.setAgentless(useAgentless)
-    spanWriter?.setAgentless(useAgentless)
+    // A disable can happen while transport selection is still pending. Keep
+    // configuring these writers so their queued lifecycle flushes can drain.
+    currentEvalWriter.setAgentless(useAgentless)
+    currentSpanWriter.setAgentless(useAgentless)
 
     telemetry.recordLLMObsEnabled(startTime, config)
     log.debug('[LLMObs] Enabled LLM Observability with configuration: %o', config.llmobs)
@@ -109,19 +134,43 @@ function disable () {
   if (injectCh.hasSubscribers) injectCh.unsubscribe(handleLLMObsInjection)
   if (registerUserSpanProcessorCh.hasSubscribers) registerUserSpanProcessorCh.unsubscribe(handleRegisterProcessor)
 
-  spanWriter?.destroy()
-  evalWriter?.destroy()
+  const retiredSpanWriter = spanWriter
+  const retiredEvalWriter = evalWriter
   spanProcessor?.setWriter(null)
+  unregisterTelemetryFlusher?.()
+  unregisterTelemetryFlusher = undefined
 
   spanWriter = null
   evalWriter = null
 
+  retireWriters(retiredSpanWriter, retiredEvalWriter)
+
   log.debug('[LLMObs] Disabled LLM Observability')
 }
 
+/**
+ * Keeps retired writers reachable until their destroy-triggered deliveries complete.
+ * @param {LLMObsSpanWriter | null} retiredSpanWriter
+ * @param {LLMObsEvalMetricsWriter | null} retiredEvalWriter
+ * @returns {void}
+ */
+function retireWriters (retiredSpanWriter, retiredEvalWriter) {
+  const retiredWriters = [retiredSpanWriter, retiredEvalWriter].filter(Boolean)
+  if (retiredWriters.length === 0) return
+  let remainingWriters = retiredWriters.length
+  const unregisterRetiredFlusher = registerTelemetryFlusher(done => {
+    flushWriters(done, retiredSpanWriter, retiredEvalWriter)
+  })
+  function onWriterDestroyed () {
+    if (--remainingWriters === 0) unregisterRetiredFlusher?.()
+  }
+  for (const writer of retiredWriters) writer.destroy(onWriterDestroyed)
+}
+
 // since LLMObs traces can extend between services and be the same trace,
 // we need to propagate the parent id, mlApp, session id, and sampling rate/decision.
-function handleLLMObsInjection ({ carrier }) {
+/** @param {TraceTagInjection} injection */
+function handleLLMObsInjection (injection) {
   // Respect the standard propagator's gate: when trace tag propagation is
   // disabled, don't write `x-datadog-tags` for LLMObs either.
   if (globalTracerConfig.DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH === 0) return
@@ -159,48 +208,62 @@ function handleLLMObsInjection ({ carrier }) {
     mlObsSpanTags, parent
   )
 
-  // `_injectTags` only writes `x-datadog-tags` when the trace has `_dd.p.*`
-  // tags, so it may be undefined here — coalesce before appending.
-  const existing = readDatadogTags(carrier)
-  let tags = existing || ''
-  if (parentId) tags += `${tags ? ',' : ''}${PROPAGATED_PARENT_ID_KEY}=${parentId}`
-  if (mlApp) tags += `${tags ? ',' : ''}${PROPAGATED_ML_APP_KEY}=${mlApp}`
-  if (sessionId) tags += `${tags ? ',' : ''}${PROPAGATED_SESSION_ID_KEY}=${sessionId}`
-  if (sampleRate != null) tags += `${tags ? ',' : ''}${PROPAGATED_SAMPLE_RATE_KEY}=${sampleRate}`
-  if (samplingDecision != null) tags += `${tags ? ',' : ''}${PROPAGATED_SAMPLING_DECISION_KEY}=${samplingDecision}`
-  if (propagatedTraceId != null) tags += `${tags ? ',' : ''}${PROPAGATED_TRACE_ID_KEY}=${propagatedTraceId}`
-  // When a local agent attribution is resolved, strip any stale upstream pagent entries that
-  // `_injectTags` may have already written into the carrier (it propagates all `_dd.p.*` from
-  // `_trace.tags`). This ensures the downstream sees a consistent id-only or id+name pair
-  // rather than a mix from different hops. The id is always digit-safe; an unsafe name is wiped.
+  const traceTagReplacements = []
+  if (parentId) traceTagReplacements.push(PROPAGATED_PARENT_ID_KEY, parentId)
+  if (mlApp) traceTagReplacements.push(PROPAGATED_ML_APP_KEY, mlApp)
+  if (sessionId) traceTagReplacements.push(PROPAGATED_SESSION_ID_KEY, sessionId)
+  if (sampleRate != null) traceTagReplacements.push(PROPAGATED_SAMPLE_RATE_KEY, sampleRate.toString())
+  if (samplingDecision != null) {
+    traceTagReplacements.push(PROPAGATED_SAMPLING_DECISION_KEY, samplingDecision.toString())
+  }
+  if (propagatedTraceId != null) traceTagReplacements.push(PROPAGATED_TRACE_ID_KEY, propagatedTraceId)
+
+  let optionalTraceTagCount = 0
   if (parentAgentSpanId) {
-    tags = stripTagsetEntry(tags, PROPAGATED_PARENT_AGENT_ID_KEY)
-    tags = stripTagsetEntry(tags, PROPAGATED_PARENT_AGENT_NAME_KEY)
+    traceTagReplacements.push(PROPAGATED_PARENT_AGENT_ID_KEY, parentAgentSpanId)
+    optionalTraceTagCount++
+    if (parentAgentName && agentNameWireSafe(parentAgentName)) {
+      traceTagReplacements.push(PROPAGATED_PARENT_AGENT_NAME_KEY, parentAgentName)
+    } else {
+      traceTagReplacements.push(PROPAGATED_PARENT_AGENT_NAME_KEY, undefined)
+    }
+    optionalTraceTagCount++
   }
-  const maxLength = globalTracerConfig.DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH
-  const tagsWithId = appendOptionalPropagatedTag(
-    tags, PROPAGATED_PARENT_AGENT_ID_KEY, parentAgentSpanId, null, maxLength
-  )
-  // Only append the name when the id fit: a name without an id is unresolvable by the backend.
-  if (tagsWithId === tags) {
-    tags = tagsWithId
-  } else {
-    tags = appendOptionalPropagatedTag(
-      tagsWithId, PROPAGATED_PARENT_AGENT_NAME_KEY, parentAgentName, agentNameWireSafe, maxLength
-    )
+
+  injection.traceTagReplacements = traceTagReplacements
+  injection.optionalTraceTagCount = optionalTraceTagCount
+}
+
+/**
+ * Flushes the specified LLMObs writers and joins deliveries active at the boundary.
+ * @param {Function} [done]
+ * @param {LLMObsSpanWriter | null} [currentSpanWriter]
+ * @param {LLMObsEvalMetricsWriter | null} [currentEvalWriter]
+ * @returns {boolean} `true` when a writer throws synchronously.
+ */
+function flushWriters (done, currentSpanWriter = spanWriter, currentEvalWriter = evalWriter) {
+  let failed = false
+  const deliveryTracker = createServerlessDeliveryTracker()
+  const flush = writer => {
+    try {
+      if (deliveryTracker && writer) deliveryTracker.track(complete => writer.flush(complete))
+      // Non-serverless flushes retain the existing writer behavior.
+      else writer?.flush()
+    } catch (error) {
+      failed = true
+      log.warn('Failed to flush LLMObs writer:', error.message)
+    }
   }
-  if (tags !== existing) writeDatadogTags(carrier, tags)
+
+  flush(currentSpanWriter)
+  flush(currentEvalWriter)
+  deliveryTracker?.waitForIdle(done)
+  if (!deliveryTracker) done?.()
+  return failed
 }
 
 function handleFlush () {
-  let err = ''
-  try {
-    spanWriter.flush()
-    evalWriter.flush()
-  } catch (e) {
-    err = 'writer_flush_error'
-    log.warn('Failed to flush LLMObs spans and evaluation metrics:', e.message)
-  }
+  const err = flushWriters() ? 'writer_flush_error' : ''
   telemetry.recordUserFlush(err)
 }
 

package/packages/dd-trace/src/llmobs/noop.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const NoopExperiments = require('./experiments/noop')
+let NoopExperiments
 
 class NoopLLMObs {
   constructor (noopTracer) {
@@ -12,6 +12,7 @@ class NoopLLMObs {
   }
 
   get experiments () {
+    NoopExperiments ??= require('./experiments/noop')
     return new NoopExperiments('LLM Observability is not enabled')
   }