dd-trace 6.7.0 → 6.8.0

package/packages/dd-trace/src/llmobs/experiments/noop.js

@@ -1,8 +1,90 @@
 'use strict'
 
+const log = require('../../log')
+
+class NoopDataset {
+  #name
+  #records
+
+  constructor (name = '', options = {}) {
+    this.#name = name
+    this.#records = (typeof options === 'string' ? [] : (options.records ?? [])).map(record => ({
+      id: record.id ?? null,
+      input: record.inputData,
+      expectedOutput: record.expectedOutput ?? null,
+      metadata: record.metadata ?? {},
+    }))
+  }
+
+  addRecord (input, expectedOutput, metadata) {
+    this.#records.push({ id: null, input, expectedOutput: expectedOutput ?? null, metadata: metadata ?? {} })
+    return this
+  }
+
+  push () {
+    return Promise.resolve({ pushedCount: 0, totalCount: 0 })
+  }
+
+  name () {
+    return this.#name
+  }
+
+  id () {
+    return null
+  }
+
+  projectId () {
+    return null
+  }
+
+  version () {
+    return null
+  }
+
+  latestVersion () {
+    return null
+  }
+
+  records () {
+    return [...this.#records]
+  }
+
+  recordIds () {
+    return []
+  }
+
+  url () {
+    return null
+  }
+}
+
+class NoopExperiment {
+  #name
+
+  constructor (name = '') {
+    this.#name = name
+  }
+
+  name () {
+    return this.#name
+  }
+
+  experimentId () {
+    return null
+  }
+
+  url () {
+    return null
+  }
+
+  run () {
+    return Promise.resolve({ experimentId: null, rows: [], url: null })
+  }
+}
+
 // No-op Experiments used when LLM Observability is disabled or the API/APP keys
-// are not configured. Every operation throws a clear, actionable error rather
-// than silently doing nothing, so misconfiguration surfaces immediately.
+// are not configured. Operations warn and return inert objects rather than
+// throwing, so intentionally disabled experiments remain graceful.
 class NoopExperiments {
   #reason
 
@@ -10,20 +92,23 @@ class NoopExperiments {
     this.#reason = reason || 'LLMObs experiments are not available'
   }
 
-  #unavailable () {
-    return new Error(`LLMObs experiments unavailable: ${this.#reason}`)
+  #warn () {
+    log.warn('LLMObs experiments unavailable: %s', this.#reason)
   }
 
-  createDataset () {
-    throw this.#unavailable()
+  createDataset (name, options = {}) {
+    this.#warn()
+    return new NoopDataset(name, options)
   }
 
-  pullDataset () {
-    return Promise.reject(this.#unavailable())
+  pullDataset (name) {
+    this.#warn()
+    return Promise.resolve(new NoopDataset(name))
   }
 
-  experiment () {
-    throw this.#unavailable()
+  experiment (options = {}) {
+    this.#warn()
+    return new NoopExperiment(options.name)
   }
 }
 

package/packages/dd-trace/src/llmobs/experiments/result.js

@@ -22,13 +22,24 @@ class Row {
   }
 }
 
+class ExperimentRun {
+  constructor (fields) {
+    this.runId = fields.runId
+    this.runIteration = fields.runIteration
+    this.rows = fields.rows
+    this.summaryEvaluations = fields.summaryEvaluations
+  }
+}
+
 // Returned by Experiment.run().
 class ExperimentResult {
-  constructor (experimentId, rows, url) {
+  constructor (experimentId, rows, url, runs = [], summaryEvaluations = {}) {
     this.experimentId = experimentId
     this.rows = rows
     this.url = url
+    this.runs = runs
+    this.summaryEvaluations = summaryEvaluations
   }
 }
 
-module.exports = { Row, ExperimentResult }
+module.exports = { Row, ExperimentRun, ExperimentResult }

package/packages/dd-trace/src/llmobs/experiments/util.js

@@ -0,0 +1,218 @@
+'use strict'
+
+const log = require('../../log')
+
+// Matches the backend and dd-trace-py evaluator metric label contract.
+const EVALUATOR_NAME_PATTERN = /^[a-zA-Z0-9_-]+$/
+
+/**
+ * @typedef {null | string | number | boolean | NormalizedJsonValue[] | { [key: string]: NormalizedJsonValue }}
+ *   NormalizedJsonValue
+ */
+
+/**
+ * @param {object | null | undefined} value
+ * @returns {boolean}
+ */
+function hasEntries (value) {
+  if (!value) return false
+  for (const key of Object.keys(value)) {
+    if (Object.hasOwn(value, key)) return true
+  }
+  return false
+}
+
+/**
+ * @param {string} name
+ */
+function validateEvaluatorName (name) {
+  if (typeof name !== 'string') throw new TypeError('Evaluator name must be a string')
+  if (name.length === 0) throw new Error('Evaluator name cannot be empty')
+  if (!EVALUATOR_NAME_PATTERN.test(name)) {
+    throw new Error(
+      `Evaluator name '${name}' is invalid. Name must contain only alphanumeric characters, underscores, and hyphens.`
+    )
+  }
+}
+
+/**
+ * @param {(...args: unknown[]) => unknown} fn
+ * @param {string} fallback
+ * @returns {string}
+ */
+function functionName (fn, fallback) {
+  return typeof fn.name === 'string' && fn.name.length > 0 ? fn.name : fallback
+}
+
+/**
+ * @param {unknown} evaluators
+ * @param {string} kind
+ * @returns {Array<[string, (...args: unknown[]) => unknown]>}
+ */
+function normalizeEvaluators (evaluators, kind) {
+  if (evaluators == null) return []
+
+  const normalized = []
+  if (Array.isArray(evaluators)) {
+    const indexesByName = new Map()
+    for (let i = 0; i < evaluators.length; i++) {
+      const evaluator = evaluators[i]
+      if (typeof evaluator !== 'function') throw new TypeError(`${kind} evaluator must be a function`)
+      const name = functionName(evaluator, `${kind}_evaluator_${i}`)
+      validateEvaluatorName(name)
+      if (indexesByName.has(name)) {
+        log.warn('Duplicate %s evaluator name %s; previous evaluator will be overwritten', kind, name)
+        normalized[indexesByName.get(name)] = [name, evaluator]
+      } else {
+        indexesByName.set(name, normalized.length)
+        normalized.push([name, evaluator])
+      }
+    }
+    return normalized
+  }
+
+  if (typeof evaluators !== 'object') {
+    throw new TypeError(`${kind} evaluators must be an array of functions or an object keyed by evaluator name`)
+  }
+
+  for (const [name, evaluator] of Object.entries(evaluators)) {
+    validateEvaluatorName(name)
+    if (typeof evaluator !== 'function') throw new TypeError(`${kind} evaluator '${name}' must be a function`)
+    normalized.push([name, evaluator])
+  }
+  return normalized
+}
+
+/**
+ * @param {unknown} value
+ * @returns {string}
+ */
+function inferMetricType (value) {
+  if (typeof value === 'boolean') return 'boolean'
+  if (typeof value === 'number' && Number.isFinite(value)) return 'score'
+  if (typeof value === 'string') return 'categorical'
+  return 'json'
+}
+
+/**
+ * @param {unknown} value
+ * @param {WeakSet<object>} seen
+ * @returns {NormalizedJsonValue}
+ */
+function normalizeJsonValue (value, seen) {
+  if (value === null) return null
+
+  const valueType = typeof value
+  if (valueType === 'string' || valueType === 'boolean') return value
+  if (valueType === 'number') return Number.isFinite(value) ? value : String(value)
+  if (valueType === 'bigint') return value.toString()
+  if (valueType === 'undefined') return null
+  if (valueType === 'symbol') return value.toString()
+  if (valueType === 'function') return { type: 'function', name: value.name || '<anonymous>' }
+
+  if (value instanceof Date) return value.toISOString()
+  if (value instanceof Error) {
+    return { type: value.name, message: value.message, stack: value.stack ?? '' }
+  }
+
+  if (seen.has(value)) return '[Circular]'
+  seen.add(value)
+
+  if (Array.isArray(value)) {
+    const normalized = new Array(value.length)
+    for (let i = 0; i < value.length; i++) normalized[i] = normalizeJsonValue(value[i], seen)
+    seen.delete(value)
+    return normalized
+  }
+
+  const normalized = {}
+  for (const [key, nestedValue] of Object.entries(value)) {
+    normalized[key] = normalizeJsonValue(nestedValue, seen)
+  }
+  seen.delete(value)
+  return normalized
+}
+
+/**
+ * @param {unknown} value
+ * @returns {{ [key: string]: NormalizedJsonValue }}
+ */
+function normalizeJsonMetricValue (value) {
+  const normalized = normalizeJsonValue(value, new WeakSet())
+  if (normalized !== null && typeof normalized === 'object' && !Array.isArray(normalized)) return normalized
+  return { value: normalized }
+}
+
+/**
+ * @param {unknown} value
+ * @returns {string}
+ */
+function stringify (value) {
+  if (value == null) return ''
+  if (typeof value === 'string') return value.toLowerCase()
+  if (typeof value === 'object') {
+    try {
+      const stringified = JSON.stringify(value)
+      if (stringified !== undefined) return stringified.toLowerCase()
+    } catch {}
+  }
+  return String(value).toLowerCase()
+}
+
+/**
+ * @param {Record<string, unknown> | undefined} userTags
+ * @param {Record<string, unknown>} autoTags
+ * @returns {string[]}
+ */
+function buildTags (userTags, autoTags) {
+  const tags = new Map()
+  for (const [key, value] of Object.entries(userTags ?? {})) {
+    tags.set(key, `${key}:${value}`)
+  }
+  for (const [key, value] of Object.entries(autoTags)) {
+    if (value !== undefined && value !== null && value !== '') tags.set(key, `${key}:${value}`)
+  }
+  return [...tags.values()]
+}
+
+/**
+ * @param {Record<string, unknown> | undefined} userTags
+ * @param {Record<string, unknown>} autoTags
+ * @returns {Record<string, unknown>}
+ */
+function buildExperimentTagObject (userTags, autoTags) {
+  return userTags ? { ...userTags, ...autoTags } : { ...autoTags }
+}
+
+/**
+ * @param {number} ms
+ * @returns {Promise<void>}
+ */
+function sleep (ms) {
+  if (ms <= 0) return Promise.resolve()
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+/**
+ * @param {Record<string, unknown> | undefined} recordMetadata
+ * @param {Record<string, unknown>} config
+ * @returns {Record<string, unknown>}
+ */
+function buildSpanMetadata (recordMetadata, config) {
+  return recordMetadata
+    ? { ...recordMetadata, experiment_config: config }
+    : { experiment_config: config }
+}
+
+module.exports = {
+  buildExperimentTagObject,
+  buildSpanMetadata,
+  buildTags,
+  hasEntries,
+  inferMetricType,
+  normalizeEvaluators,
+  normalizeJsonMetricValue,
+  sleep,
+  stringify,
+  validateEvaluatorName,
+}

package/packages/dd-trace/src/llmobs/plugins/anthropic/index.js

@@ -93,7 +93,7 @@ class AnthropicLLMObsPlugin extends LLMObsPlugin {
 
             const { usage } = chunk
             if (usage) {
-              const responseUsage = response.usage ?? (response.usage = { input_tokens: 0, output_tokens: 0 })
+              const responseUsage = (response.usage ??= { input_tokens: 0, output_tokens: 0 })
               responseUsage.output_tokens = usage.output_tokens
 
               const cacheCreationTokens = usage.cache_creation_input_tokens

package/packages/dd-trace/src/llmobs/plugins/claude-agent-sdk/index.js

@@ -8,7 +8,7 @@ const { splitModel } = require('../../../../../datadog-plugin-claude-agent-sdk/s
 const subagentToolIds = new Set()
 
 function normalizeToolOutputString (raw) {
-  const footerIndex = raw.search(/\s*agentId: [^\s]+ \(use SendMessage\b/)
+  const footerIndex = raw.search(/agentId: \S+ \(use SendMessage\b/)
   if (footerIndex === -1) return raw
 
   return raw.slice(0, footerIndex).trimEnd()

package/packages/dd-trace/src/llmobs/plugins/genai/index.js

@@ -24,7 +24,7 @@ class GenAiLLMObsPlugin extends LLMObsPlugin {
     // Subscribe to streaming chunk events
     this.addSub('apm:google:genai:request:chunk', ({ ctx, chunk, done }) => {
       ctx.isStreaming = true
-      ctx.chunks = ctx.chunks || []
+      ctx.chunks ||= []
 
       if (chunk) ctx.chunks.push(chunk)
       if (!done) return

package/packages/dd-trace/src/llmobs/plugins/langchain/handlers/index.js

@@ -36,7 +36,7 @@ class LangChainLLMObsHandler {
     const runIdBase = runId ? runId.split('-').slice(0, -1).join('-') : ''
 
     const responseMetadata = message.response_metadata || {}
-    usage = usage || responseMetadata.usage || responseMetadata.tokenUsage || {}
+    usage ||= responseMetadata.usage || responseMetadata.tokenUsage || {}
 
     const inputTokens = usage.promptTokens || usage.inputTokens || usage.prompt_tokens || usage.input_tokens || 0
     const outputTokens =

package/packages/dd-trace/src/llmobs/sdk.js

@@ -35,12 +35,6 @@ class LLMObs extends NoopLLMObs {
    */
   #hasUserSpanProcessor = false
 
-  /**
-   * Lazily-created experiments facade (see ./experiments).
-   * @type {import('./experiments').Experiments | undefined}
-   */
-  #experiments
-
   /**
    * @param {import('../tracer')} tracer - Tracer instance
    * @param {import('./index')} llmobsModule - LLMObs module instance
@@ -66,8 +60,7 @@ class LLMObs extends NoopLLMObs {
    * a clear message on use.
    */
   get experiments () {
-    this.#experiments ??= createExperiments(this._config)
-    return this.#experiments
+    return createExperiments(this._config, this)
   }
 
   enable (options = {}) {
@@ -318,14 +311,14 @@ class LLMObs extends NoopLLMObs {
       }
       throw e
     } finally {
-      if (autoinstrumented === false) {
+      if (!autoinstrumented) {
         telemetry.recordLLMObsAnnotate(span, err)
       }
     }
   }
 
   exportSpan (span) {
-    span = span || this._active()
+    span ||= this._active()
     let err = ''
     try {
       if (!span) {
@@ -426,7 +419,7 @@ class LLMObs extends NoopLLMObs {
         err = 'invalid_metric_value'
         throw new Error('value must be a boolean for a boolean metric')
       }
-      if (metricType === 'json' && !(typeof value === 'object' && value != null && !Array.isArray(value))) {
+      if (metricType === 'json' && (typeof value !== 'object' || value == null || Array.isArray(value))) {
         err = 'invalid_metric_value'
         throw new Error('value must be a JSON object for a json metric')
       }

package/packages/dd-trace/src/llmobs/span_processor.js

@@ -239,6 +239,14 @@ class LLMObsSpanProcessor {
 
     const apmTraceId = span.context().toTraceId(true)
     const llmobsTraceId = mlObsTags[TRACE_ID] ?? apmTraceId
+    const dd = {
+      span_id: span.context().toSpanId(),
+      trace_id: apmTraceId,
+      sample_rate: mlObsTags[SAMPLE_RATE],
+      sampling_decision: mlObsTags[SAMPLING_DECISION],
+      apm_trace_id: apmTraceId,
+    }
+    if (tags.experiment_id) dd.scope = 'experiments'
 
     const llmObsSpanEvent = {
       trace_id: llmobsTraceId,
@@ -251,13 +259,7 @@ class LLMObsSpanProcessor {
       status: error ? 'error' : 'ok',
       meta,
       metrics,
-      _dd: {
-        span_id: span.context().toSpanId(),
-        trace_id: apmTraceId,
-        sample_rate: mlObsTags[SAMPLE_RATE],
-        sampling_decision: mlObsTags[SAMPLING_DECISION],
-        apm_trace_id: apmTraceId,
-      },
+      _dd: dd,
     }
 
     if (sessionId) llmObsSpanEvent.session_id = sessionId

package/packages/dd-trace/src/llmobs/tagger.js

@@ -158,8 +158,7 @@ class LLMObsTagger {
     if (modelName) this.tagModelName(span, modelName)
     if (modelProvider) this._setTag(span, MODEL_PROVIDER, modelProvider)
 
-    sessionId = sessionId ||
-      registry.get(parent)?.[SESSION_ID] ||
+    sessionId ||= registry.get(parent)?.[SESSION_ID] ||
       traceTags[SESSION_ID_TRACE_DEFAULT_KEY] ||
       traceTags[PROPAGATED_SESSION_ID_KEY]
     if (sessionId) this._setTag(span, SESSION_ID, sessionId)

package/packages/dd-trace/src/llmobs/writers/spans.js

@@ -49,12 +49,18 @@ class LLMObsSpanWriter extends BaseWriter {
   }
 
   makePayload (events) {
-    return events.map(event => ({
-      '_dd.stage': 'raw',
-      '_dd.tracer_version': tracerVersion,
-      event_type: this._eventType,
-      spans: [event],
-    }))
+    return events.map(event => {
+      const eventData = {
+        '_dd.stage': 'raw',
+        '_dd.tracer_version': tracerVersion,
+        event_type: this._eventType,
+        spans: [event],
+      }
+      if (event?._dd?.scope === 'experiments') {
+        eventData['_dd.scope'] = 'experiments'
+      }
+      return eventData
+    })
   }
 
   _truncateSpanEvent (event) {

package/packages/dd-trace/src/noop/proxy.js

@@ -55,7 +55,7 @@ class NoopProxy {
 
     if (typeof fn !== 'function') return
 
-    options = options || {}
+    options ||= {}
 
     return this._tracer.trace(name, options, fn)
   }
@@ -68,7 +68,7 @@ class NoopProxy {
 
     if (typeof fn !== 'function') return fn
 
-    options = options || {}
+    options ||= {}
 
     return this._tracer.wrap(name, options, fn)
   }

package/packages/dd-trace/src/opentelemetry/metrics/otlp_transformer.js

@@ -198,7 +198,7 @@ class OtlpTransformer extends OtlpTransformerBase {
     if (isJson) {
       dataPoint.startTimeUnixNano = String(dataPoint.startTimeUnixNano)
       dataPoint.timeUnixNano = String(dataPoint.timeUnixNano)
-      dataPoint.count = dataPoint.count || 0
+      dataPoint.count ||= 0
     }
 
     return dataPoint

package/packages/dd-trace/src/opentelemetry/metrics/periodic_metric_reader.js

@@ -253,7 +253,8 @@ class PeriodicMetricReader {
   #collectAndExport (callback = () => {}) {
     // Atomically drain measurements for export. New measurements can be recorded
     // during export without interfering with this batch.
-    const allMeasurements = this.#measurements.splice(0)
+    const allMeasurements = this.#measurements
+    this.#measurements = []
 
     for (const instrument of this.observableInstruments) {
       const observableMeasurements = instrument.collect()

package/packages/dd-trace/src/opentracing/span.js

@@ -440,7 +440,7 @@ class DatadogSpan {
       }
     }
 
-    spanContext._trace.ticks = spanContext._trace.ticks || now()
+    spanContext._trace.ticks ||= now()
     if (startTime) {
       spanContext._trace.startTime = startTime
     }

package/packages/dd-trace/src/opentracing/span_context.js

@@ -8,7 +8,7 @@ const TRACE_ID_128 = '_dd.p.tid'
 
 class DatadogSpanContext {
   constructor (props) {
-    props = props || {}
+    props ||= {}
 
     this._traceId = props.traceId
     this._spanId = props.spanId