dd-trace 6.4.0 → 6.5.0

package/packages/dd-trace/src/llmobs/experiments/experiment.js

@@ -0,0 +1,283 @@
+'use strict'
+
+const id = require('../../id')
+
+const { API_BASE_PATH } = require('./client')
+const { Row, ExperimentResult } = require('./result')
+
+// Mirrors dd-trace-py's _generate_metric_from_evaluation: plain objects are
+// json, everything else falls through to the lowercased categorical fallback.
+function inferMetricType (value) {
+  if (typeof value === 'boolean') return 'boolean'
+  if (typeof value === 'number' && Number.isFinite(value)) return 'score'
+  if (value !== null && typeof value === 'object' && !Array.isArray(value)) return 'json'
+  return 'categorical'
+}
+
+function stringify (value) {
+  if (value === null || value === undefined) return ''
+  if (typeof value === 'string') return value.toLowerCase()
+  if (typeof value === 'object') return JSON.stringify(value).toLowerCase()
+  return String(value).toLowerCase()
+}
+
+// Build the tag list, letting auto tags win over user tags on key conflict.
+function buildTags (userTags, autoTags) {
+  const tags = new Map()
+  for (const [k, v] of Object.entries(userTags ?? {})) {
+    tags.set(k, `${k}:${v}`)
+  }
+  for (const [k, v] of Object.entries(autoTags)) {
+    tags.set(k, `${k}:${v}`)
+  }
+  return [...tags.values()]
+}
+
+// One span per experiment row (LLM Obs experiment span wire format).
+function toSpan (row, metadata, ids, spanName, userTags) {
+  const meta = {
+    input: row.input ?? null,
+    output: row.output ?? null,
+    expected_output: row.expectedOutput ?? null,
+  }
+  if (metadata && Object.keys(metadata).length > 0) {
+    meta.metadata = metadata
+  }
+  if (row.isError) {
+    meta.error = { type: row.errorType ?? '', message: row.errorMessage ?? '', stack: '' }
+  }
+
+  return {
+    span_id: row.spanId,
+    trace_id: row.traceId,
+    project_id: ids.projectId,
+    dataset_id: ids.datasetId,
+    name: spanName,
+    start_ns: row.startNs,
+    duration: row.durationNs,
+    status: row.isError ? 'error' : 'ok',
+    meta,
+    tags: buildTags(userTags, {
+      experiment_id: ids.experimentId,
+      dataset_id: ids.datasetId,
+      dataset_record_id: ids.datasetRecordId,
+    }),
+  }
+}
+
+// One metric per evaluator per row.
+function toMetric (label, value, errorMessage, spanId, timestampMs, experimentId, userTags) {
+  const metric = {
+    label,
+    span_id: spanId,
+    timestamp_ms: timestampMs,
+    tags: buildTags(userTags, { experiment_id: experimentId }),
+  }
+
+  if (errorMessage !== null) {
+    metric.metric_type = 'categorical'
+    metric.error = { message: errorMessage }
+    return metric
+  }
+
+  const type = inferMetricType(value)
+  metric.metric_type = type
+  if (type === 'boolean') metric.boolean_value = value
+  else if (type === 'score') metric.score_value = value
+  else if (type === 'json') metric.json_value = value
+  else metric.categorical_value = stringify(value)
+  return metric
+}
+
+// Builder + run() orchestration: runs rows sequentially, emits one root span
+// per dataset row, and posts all spans + metrics in a single events call.
+class Experiment {
+  #client
+  #name
+  #description
+  #dataset
+  #task
+  #evaluators
+  #config
+  #tags
+  #experimentId
+
+  constructor (client, options = {}) {
+    if (!options.name) throw new Error('Experiment name is required')
+    if (!options.dataset) throw new Error('Experiment dataset is required')
+    if (typeof options.task !== 'function') throw new Error('Experiment task is required')
+
+    this.#client = client
+    this.#name = options.name
+    this.#description = options.description ?? ''
+    this.#dataset = options.dataset
+    this.#task = options.task
+    this.#evaluators = new Map(Object.entries(options.evaluators ?? {}))
+    this.#config = { ...options.config }
+    this.#tags = { ...options.tags }
+    this.#experimentId = null
+  }
+
+  name () {
+    return this.#name
+  }
+
+  experimentId () {
+    return this.#experimentId
+  }
+
+  url () {
+    if (this.#experimentId === null) return null
+    return `${this.#client.appBase}/llm/experiments/${this.#experimentId}`
+  }
+
+  async run () {
+    const projectId = await this.#client.ensureProjectId()
+
+    await this.#dataset.ensureCreatedAndPushed(projectId)
+    const datasetId = this.#dataset.id()
+    if (datasetId === null) {
+      throw new Error(`Dataset '${this.#dataset.name()}' has no id after push`)
+    }
+
+    // Create the experiment. ensure_unique makes the backend mint a fresh
+    // experiment under the project on every run.
+    const attributes = {
+      name: this.#name,
+      project_id: projectId,
+      dataset_id: datasetId,
+      description: this.#description,
+      ensure_unique: true,
+    }
+    if (Object.keys(this.#config).length > 0) attributes.config = this.#config
+
+    let created
+    try {
+      created = await this.#client.request('POST', `${API_BASE_PATH}/experiments`, {
+        data: { type: 'experiments', attributes },
+      })
+    } catch (err) {
+      throw new Error(`Failed to create experiment '${this.#name}': ${err.message}`)
+    }
+    this.#experimentId = created?.data?.id ?? null
+    const experimentId = this.#experimentId
+
+    try {
+      const records = this.#dataset.records()
+      const recordIds = this.#dataset.recordIds()
+      const rows = []
+      const spans = []
+      const metrics = []
+      let hasRowError = false
+
+      for (let i = 0; i < records.length; i++) {
+        const record = records[i]
+        const datasetRecordId = i < recordIds.length ? recordIds[i] : ''
+        const startNs = Date.now() * 1e6
+        const startHr = process.hrtime.bigint()
+
+        // Root-span id generation, same convention as opentracing/span.js: a single
+        // random 64-bit id for the span, reused as the trace id's low 64 bits with a
+        // start-time-derived high 64 bits (like the `_dd.p.tid` 128-bit trace id tag).
+        const spanIdentifier = id()
+        const traceIdHigh = Math.floor(startNs / 1e9).toString(16).padStart(8, '0').padEnd(16, '0')
+        const spanId = spanIdentifier.toString(16).padStart(16, '0')
+        const traceId = spanIdentifier.toTraceIdHex(traceIdHigh).padStart(32, '0')
+
+        let output = null
+        let errorType = null
+        let errorMessage = null
+        try {
+          // Rows run one at a time by design.
+          // eslint-disable-next-line no-await-in-loop
+          output = await this.#task(record.input, this.#config)
+        } catch (err) {
+          errorType = err.name || 'Error'
+          errorMessage = err.message ?? String(err)
+        }
+
+        const durationNs = Number(process.hrtime.bigint() - startHr)
+        const evaluations = {}
+        const evaluationErrors = {}
+        const timestampMs = Date.now()
+
+        for (const [label, evaluator] of this.#evaluators) {
+          if (errorType !== null) {
+            const msg = 'task error; evaluation skipped'
+            evaluationErrors[label] = msg
+            metrics.push(toMetric(label, null, msg, spanId, timestampMs, experimentId, this.#tags))
+            continue
+          }
+          try {
+            // eslint-disable-next-line no-await-in-loop
+            const value = await evaluator(record.input, output, record.expectedOutput)
+            evaluations[label] = value
+            metrics.push(toMetric(label, value, null, spanId, timestampMs, experimentId, this.#tags))
+          } catch (err) {
+            const msg = err.message ?? String(err)
+            evaluationErrors[label] = msg
+            metrics.push(toMetric(label, null, msg, spanId, timestampMs, experimentId, this.#tags))
+          }
+        }
+
+        const row = new Row({
+          index: i,
+          spanId,
+          traceId,
+          startNs,
+          durationNs,
+          input: record.input,
+          output,
+          expectedOutput: record.expectedOutput,
+          errorType,
+          errorMessage,
+          evaluations,
+          evaluationErrors,
+        })
+        rows.push(row)
+        if (row.isError || Object.keys(evaluationErrors).length > 0) hasRowError = true
+        spans.push(toSpan(row, record.metadata, {
+          experimentId,
+          projectId,
+          datasetId,
+          datasetRecordId,
+        }, this.#name, this.#tags))
+      }
+
+      await this.#postEvents(experimentId, spans, metrics)
+      // A row error doesn't abort the run, but the experiment didn't succeed cleanly.
+      await this.#updateStatus(
+        experimentId,
+        hasRowError ? 'failed' : 'completed',
+        hasRowError ? 'one or more rows failed' : null
+      )
+
+      return new ExperimentResult(experimentId, rows, this.url())
+    } catch (err) {
+      await this.#updateStatus(experimentId, 'failed', err.message ?? String(err))
+      throw err
+    }
+  }
+
+  async #postEvents (experimentId, spans, metrics) {
+    await this.#client.request('POST', `${API_BASE_PATH}/experiments/${experimentId}/events`, {
+      data: { type: 'experiments', attributes: { spans, metrics } },
+    })
+  }
+
+  async #updateStatus (experimentId, status, error) {
+    if (!experimentId) return
+    // The experiment-update model has no status field, so this is a direct PATCH.
+    const attributes = { status }
+    if (error !== null) attributes.error = error
+    try {
+      await this.#client.request('PATCH', `${API_BASE_PATH}/experiments/${experimentId}`, {
+        data: { type: 'experiments', attributes },
+      })
+    } catch {
+      // Status update is best-effort; never let it mask the real result/error.
+    }
+  }
+}
+
+module.exports = { Experiment }

package/packages/dd-trace/src/llmobs/experiments/index.js

@@ -0,0 +1,152 @@
+'use strict'
+
+const log = require('../../log')
+const { ExperimentsClient, API_BASE_PATH } = require('./client')
+const { Dataset, DatasetRecord } = require('./dataset')
+const { Experiment } = require('./experiment')
+const NoopExperiments = require('./noop')
+
+// Poll `attempt` with exponential backoff until it returns true or the time
+// budget is spent. Used for eventually-consistent reads (pullDataset).
+async function retryWithBackoff (attempt, { maxTotalMs = 30_000, baseDelayMs = 250, maxDelayMs = 8000 } = {}) {
+  const start = Date.now()
+  let delay = baseDelayMs
+  for (;;) {
+    // eslint-disable-next-line no-await-in-loop
+    if (await attempt()) return true
+    const remaining = maxTotalMs - (Date.now() - start)
+    if (remaining <= 0) return false
+    // eslint-disable-next-line no-await-in-loop
+    await new Promise((resolve) => setTimeout(resolve, Math.min(delay, maxDelayMs, remaining)))
+    delay *= 2
+  }
+}
+
+// Entry point exposed as `tracer.llmobs.experiments`. Builds datasets and runs
+// experiments against the LLM Obs backend using the tracer's own config.
+class Experiments {
+  #client
+  #projectName
+
+  constructor (config) {
+    this.#projectName = config.llmobs?.mlApp || config.service
+    this.#client = new ExperimentsClient({
+      apiKey: config.DD_API_KEY,
+      appKey: config.DD_APP_KEY,
+      site: config.site,
+      projectName: this.#projectName,
+    })
+  }
+
+  // Create a local dataset buffer. Pushed remotely on first experiment run.
+  createDataset (name, description = '') {
+    return new Dataset(this.#client, name, description)
+  }
+
+  // Pull an existing dataset by name (with its records). Polls with exponential
+  // backoff to absorb read-after-write lag; pass `expectedRecordCount` to also
+  // wait until that many records are readable.
+  async pullDataset (name, options = {}) {
+    const { expectedRecordCount, maxWaitMs = 30_000 } = options
+    const projectId = await this.#client.ensureProjectId()
+
+    let datasetId = null
+    let description = ''
+    let records = []
+    let recordIds = []
+    let lastError = ''
+
+    const succeeded = await retryWithBackoff(async () => {
+      try {
+        if (datasetId === null) {
+          const listed = await this.#client.request(
+            'GET',
+            `${API_BASE_PATH}/${projectId}/datasets?filter[name]=${encodeURIComponent(name)}`
+          )
+          for (const item of listed?.data ?? []) {
+            if (item?.attributes?.name === name) {
+              datasetId = String(item?.id ?? '')
+              description = String(item?.attributes?.description ?? '')
+              break
+            }
+          }
+          if (datasetId === null) return false
+        }
+
+        const recs = []
+        const ids = []
+        let cursor = ''
+        // Follow the meta.after / page[cursor] pagination until the last page.
+        for (;;) {
+          const query = cursor ? `?page[cursor]=${encodeURIComponent(cursor)}` : ''
+          // eslint-disable-next-line no-await-in-loop
+          const resp = await this.#client.request(
+            'GET',
+            `${API_BASE_PATH}/${projectId}/datasets/${datasetId}/records${query}`
+          )
+          for (const item of resp?.data ?? []) {
+            const attrs = item?.attributes ?? item
+            recs.push(new DatasetRecord(attrs?.input ?? null, attrs?.expected_output ?? null, attrs?.metadata ?? {}))
+            ids.push(String(item?.id ?? ''))
+          }
+          cursor = resp?.meta?.after ?? ''
+          if (!cursor) break
+        }
+        records = recs
+        recordIds = ids
+        lastError = ''
+
+        return expectedRecordCount == null || recs.length >= expectedRecordCount
+      } catch (err) {
+        lastError = err.message
+        return false
+      }
+    }, { maxTotalMs: maxWaitMs })
+
+    if (datasetId === null && lastError) {
+      throw new Error(`Failed to list datasets in project '${this.#projectName}': ${lastError}`)
+    }
+    if (datasetId === null) {
+      throw new Error(`Dataset '${name}' not found in project '${this.#projectName}' (after ${maxWaitMs}ms)`)
+    }
+    if (!succeeded && lastError) {
+      throw new Error(`Failed to fetch records for dataset '${name}' in project '${this.#projectName}': ${lastError}`)
+    }
+    if (!succeeded && expectedRecordCount != null) {
+      throw new Error(
+        `Dataset '${name}' has ${records.length} record(s) after ${maxWaitMs}ms, expected ${expectedRecordCount} ` +
+        '— backend may not have finished ingesting the push'
+      )
+    }
+
+    return Dataset.fromExisting(this.#client, name, description, datasetId, projectId, records, recordIds)
+  }
+
+  // Build an experiment: { name, dataset, task, evaluators, description?, config?, tags? }.
+  experiment (options) {
+    return new Experiment(this.#client, options)
+  }
+}
+
+// Factory used by the LLMObs SDK: returns a real Experiments instance when
+// enabled and credentialed, otherwise a no-op that explains what's missing.
+function createExperiments (config) {
+  if (!config.llmobs?.DD_LLMOBS_ENABLED) {
+    return new NoopExperiments('LLM Observability is not enabled')
+  }
+  if (!(config.DD_API_KEY) || !config.DD_APP_KEY) {
+    log.warn('LLMObs experiments: missing api and/or app keys, set DD_API_KEY and DD_APP_KEY')
+    return new NoopExperiments('DD_API_KEY and DD_APP_KEY are required for experiments')
+  }
+  if (!config.llmobs?.mlApp && !config.service) {
+    log.warn('LLMObs experiments: no project name configured, set DD_LLMOBS_ML_APP or DD_SERVICE')
+    return new NoopExperiments(
+      'no project name configured; set the DD_LLMOBS_ML_APP environment variable (or llmobs.mlApp in ' +
+      'tracer.init()) to name the LLM Obs project, or DD_SERVICE (or service in tracer.init()) as a fallback, ' +
+      'then retry'
+    )
+  }
+  return new Experiments(config)
+}
+
+module.exports = { Experiments, createExperiments }

package/packages/dd-trace/src/llmobs/experiments/noop.js

@@ -0,0 +1,30 @@
+'use strict'
+
+// No-op Experiments used when LLM Observability is disabled or the API/APP keys
+// are not configured. Every operation throws a clear, actionable error rather
+// than silently doing nothing, so misconfiguration surfaces immediately.
+class NoopExperiments {
+  #reason
+
+  constructor (reason) {
+    this.#reason = reason || 'LLMObs experiments are not available'
+  }
+
+  #unavailable () {
+    return new Error(`LLMObs experiments unavailable: ${this.#reason}`)
+  }
+
+  createDataset () {
+    throw this.#unavailable()
+  }
+
+  pullDataset () {
+    return Promise.reject(this.#unavailable())
+  }
+
+  experiment () {
+    throw this.#unavailable()
+  }
+}
+
+module.exports = NoopExperiments

package/packages/dd-trace/src/llmobs/experiments/result.js

@@ -0,0 +1,34 @@
+'use strict'
+
+// One row of an experiment run.
+class Row {
+  constructor (fields) {
+    this.index = fields.index
+    this.spanId = fields.spanId
+    this.traceId = fields.traceId
+    this.startNs = fields.startNs
+    this.durationNs = fields.durationNs
+    this.input = fields.input
+    this.output = fields.output
+    this.expectedOutput = fields.expectedOutput
+    this.errorType = fields.errorType
+    this.errorMessage = fields.errorMessage
+    this.evaluations = fields.evaluations
+    this.evaluationErrors = fields.evaluationErrors
+  }
+
+  get isError () {
+    return this.errorType !== null
+  }
+}
+
+// Returned by Experiment.run().
+class ExperimentResult {
+  constructor (experimentId, rows, url) {
+    this.experimentId = experimentId
+    this.rows = rows
+    this.url = url
+  }
+}
+
+module.exports = { Row, ExperimentResult }

package/packages/dd-trace/src/llmobs/noop.js

@@ -1,5 +1,7 @@
 'use strict'
 
+const NoopExperiments = require('./experiments/noop')
+
 class NoopLLMObs {
   constructor (noopTracer) {
     this._tracer = noopTracer
@@ -9,6 +11,10 @@ class NoopLLMObs {
     return false
   }
 
+  get experiments () {
+    return new NoopExperiments('LLM Observability is not enabled')
+  }
+
   enable (options) {}
 
   disable () {}

package/packages/dd-trace/src/llmobs/plugins/ai/vercelTelemetry.js

@@ -13,6 +13,7 @@ const {
 const SPAN_NAME_TO_KIND_MAPPING = {
   // embeddings
   embed: 'embedding',
+  embedMany: 'embedding',
   // text generation
   generateText: 'workflow',
   streamText: 'workflow',
@@ -237,6 +238,7 @@ class VercelAiTelemetryPlugin extends BaseLLMObsPlugin {
 
     switch (operation) {
       case 'embed':
+      case 'embedMany':
         this.setEmbeddingTags(span, ctx)
         break
       case 'generateText':
@@ -260,14 +262,29 @@ class VercelAiTelemetryPlugin extends BaseLLMObsPlugin {
   setEmbeddingTags (span, ctx) {
     const { event, result } = ctx
 
+    // works for both embed and embedMany, either single string or array of strings
     const input = event.value
-    const embedding = result?.embedding
-    const embeddingTextResult = embedding ? `[1 embedding(s) returned with size ${embedding.length}]` : ''
+
+    if (!result) {
+      this._tagger.tagEmbeddingIO(span, input)
+      return
+    }
+
+    // embed returns single embedding, embedMany returns an array of embeddings
+    const singleEmbedding = result.embedding
+    const multiEmbedding = result.embeddings
+
+    let embeddingTextResult = ''
+    if (singleEmbedding) {
+      embeddingTextResult = `[1 embedding(s) returned with size ${singleEmbedding.length}]`
+    } else if (Array.isArray(multiEmbedding)) {
+      embeddingTextResult = `[${multiEmbedding.length} embedding(s) returned with size ${multiEmbedding[0]?.length}]`
+    }
 
     this._tagger.tagEmbeddingIO(span, input, embeddingTextResult)
 
     this._tagger.tagMetrics(span, {
-      inputTokens: result?.usage?.tokens,
+      inputTokens: result.usage?.tokens,
     })
   }
 

package/packages/dd-trace/src/llmobs/plugins/anthropic/index.js

@@ -59,7 +59,7 @@ class AnthropicLLMObsPlugin extends LLMObsPlugin {
             const { delta } = chunk
             if (!delta) continue
 
-            const lastBlock = response.content[response.content.length - 1]
+            const lastBlock = response.content.at(-1)
             if (!lastBlock) continue
 
             if (delta.type === 'thinking_delta') {
@@ -77,7 +77,7 @@ class AnthropicLLMObsPlugin extends LLMObsPlugin {
             break
           }
           case 'content_block_stop': {
-            const lastBlock = response.content[response.content.length - 1]
+            const lastBlock = response.content.at(-1)
             if (!lastBlock) break
             if (lastBlock.type === 'tool_use') {
               const input = lastBlock.input ?? '{}'

package/packages/dd-trace/src/llmobs/plugins/langgraph/index.js

@@ -74,7 +74,7 @@ class NextStreamLLMObsPlugin extends LLMObsPlugin {
 
     const { streamInputs: inputs, chunks } = streamData
     const input = inputs == null ? undefined : formatIO(inputs)
-    const lastChunk = chunks.length > 0 ? chunks[chunks.length - 1] : undefined
+    const lastChunk = chunks.length > 0 ? chunks.at(-1) : undefined
     const output = !hasError && lastChunk != null ? formatIO(lastChunk) : undefined
 
     this._tagger.tagTextIO(span, input, output)

package/packages/dd-trace/src/llmobs/plugins/openai/constants.js

@@ -6,6 +6,12 @@ const INPUT_TYPE_TEXT = 'input_text'
 
 const IMAGE_FALLBACK = '[image]'
 const FILE_FALLBACK = '[file]'
+const AUDIO_FALLBACK = '[audio]'
+
+// OpenAI audio `format` values that don't map cleanly to `audio/<format>`.
+const AUDIO_MIME_TYPES = {
+  mp3: 'audio/mpeg',
+}
 
 module.exports = {
   INPUT_TYPE_IMAGE,
@@ -13,4 +19,6 @@ module.exports = {
   INPUT_TYPE_TEXT,
   IMAGE_FALLBACK,
   FILE_FALLBACK,
+  AUDIO_FALLBACK,
+  AUDIO_MIME_TYPES,
 }