dd-trace 6.7.0 → 6.9.0

package/packages/dd-trace/src/llmobs/experiments/noop.js

@@ -1,8 +1,90 @@
 'use strict'
 
+const log = require('../../log')
+
+class NoopDataset {
+  #name
+  #records
+
+  constructor (name = '', options = {}) {
+    this.#name = name
+    this.#records = (typeof options === 'string' ? [] : (options.records ?? [])).map(record => ({
+      id: record.id ?? null,
+      input: record.inputData,
+      expectedOutput: record.expectedOutput ?? null,
+      metadata: record.metadata ?? {},
+    }))
+  }
+
+  addRecord (input, expectedOutput, metadata) {
+    this.#records.push({ id: null, input, expectedOutput: expectedOutput ?? null, metadata: metadata ?? {} })
+    return this
+  }
+
+  push () {
+    return Promise.resolve({ pushedCount: 0, totalCount: 0 })
+  }
+
+  name () {
+    return this.#name
+  }
+
+  id () {
+    return null
+  }
+
+  projectId () {
+    return null
+  }
+
+  version () {
+    return null
+  }
+
+  latestVersion () {
+    return null
+  }
+
+  records () {
+    return [...this.#records]
+  }
+
+  recordIds () {
+    return []
+  }
+
+  url () {
+    return null
+  }
+}
+
+class NoopExperiment {
+  #name
+
+  constructor (name = '') {
+    this.#name = name
+  }
+
+  name () {
+    return this.#name
+  }
+
+  experimentId () {
+    return null
+  }
+
+  url () {
+    return null
+  }
+
+  run () {
+    return Promise.resolve({ experimentId: null, rows: [], url: null })
+  }
+}
+
 // No-op Experiments used when LLM Observability is disabled or the API/APP keys
-// are not configured. Every operation throws a clear, actionable error rather
-// than silently doing nothing, so misconfiguration surfaces immediately.
+// are not configured. Operations warn and return inert objects rather than
+// throwing, so intentionally disabled experiments remain graceful.
 class NoopExperiments {
   #reason
 
@@ -10,20 +92,23 @@ class NoopExperiments {
     this.#reason = reason || 'LLMObs experiments are not available'
   }
 
-  #unavailable () {
-    return new Error(`LLMObs experiments unavailable: ${this.#reason}`)
+  #warn () {
+    log.warn('LLMObs experiments unavailable: %s', this.#reason)
   }
 
-  createDataset () {
-    throw this.#unavailable()
+  createDataset (name, options = {}) {
+    this.#warn()
+    return new NoopDataset(name, options)
   }
 
-  pullDataset () {
-    return Promise.reject(this.#unavailable())
+  pullDataset (name) {
+    this.#warn()
+    return Promise.resolve(new NoopDataset(name))
   }
 
-  experiment () {
-    throw this.#unavailable()
+  experiment (options = {}) {
+    this.#warn()
+    return new NoopExperiment(options.name)
   }
 }
 

package/packages/dd-trace/src/llmobs/experiments/result.js

@@ -22,13 +22,24 @@ class Row {
   }
 }
 
+class ExperimentRun {
+  constructor (fields) {
+    this.runId = fields.runId
+    this.runIteration = fields.runIteration
+    this.rows = fields.rows
+    this.summaryEvaluations = fields.summaryEvaluations
+  }
+}
+
 // Returned by Experiment.run().
 class ExperimentResult {
-  constructor (experimentId, rows, url) {
+  constructor (experimentId, rows, url, runs = [], summaryEvaluations = {}) {
     this.experimentId = experimentId
     this.rows = rows
     this.url = url
+    this.runs = runs
+    this.summaryEvaluations = summaryEvaluations
   }
 }
 
-module.exports = { Row, ExperimentResult }
+module.exports = { Row, ExperimentRun, ExperimentResult }

package/packages/dd-trace/src/llmobs/experiments/util.js

@@ -0,0 +1,220 @@
+'use strict'
+
+const log = require('../../log')
+
+// Matches the backend and dd-trace-py evaluator metric label contract.
+const EVALUATOR_NAME_PATTERN = /^[a-zA-Z0-9_-]+$/
+
+/**
+ * @typedef {null | string | number | boolean | NormalizedJsonValue[] | { [key: string]: NormalizedJsonValue }}
+ *   NormalizedJsonValue
+ */
+
+/**
+ * @param {object | null | undefined} value
+ * @returns {boolean}
+ */
+function hasEntries (value) {
+  if (!value) return false
+  for (const key of Object.keys(value)) {
+    if (Object.hasOwn(value, key)) return true
+  }
+  return false
+}
+
+/**
+ * @param {string} name
+ */
+function validateEvaluatorName (name) {
+  if (typeof name !== 'string') throw new TypeError('Evaluator name must be a string')
+  if (name.length === 0) throw new Error('Evaluator name cannot be empty')
+  if (!EVALUATOR_NAME_PATTERN.test(name)) {
+    throw new Error(
+      `Evaluator name '${name}' is invalid. Name must contain only alphanumeric characters, underscores, and hyphens.`
+    )
+  }
+}
+
+/**
+ * @param {(...args: unknown[]) => unknown} fn
+ * @param {string} fallback
+ * @returns {string}
+ */
+function functionName (fn, fallback) {
+  return typeof fn.name === 'string' && fn.name.length > 0 ? fn.name : fallback
+}
+
+/**
+ * @param {unknown} evaluators
+ * @param {string} kind
+ * @returns {Array<[string, (...args: unknown[]) => unknown]>}
+ */
+function normalizeEvaluators (evaluators, kind) {
+  if (evaluators == null) return []
+
+  const normalized = []
+  if (Array.isArray(evaluators)) {
+    const indexesByName = new Map()
+    for (let i = 0; i < evaluators.length; i++) {
+      const evaluator = evaluators[i]
+      if (typeof evaluator !== 'function') throw new TypeError(`${kind} evaluator must be a function`)
+      const name = functionName(evaluator, `${kind}_evaluator_${i}`)
+      validateEvaluatorName(name)
+      if (indexesByName.has(name)) {
+        log.warn('Duplicate %s evaluator name %s; previous evaluator will be overwritten', kind, name)
+        normalized[indexesByName.get(name)] = [name, evaluator]
+      } else {
+        indexesByName.set(name, normalized.length)
+        normalized.push([name, evaluator])
+      }
+    }
+    return normalized
+  }
+
+  if (typeof evaluators !== 'object') {
+    throw new TypeError(`${kind} evaluators must be an array of functions or an object keyed by evaluator name`)
+  }
+
+  for (const [name, evaluator] of Object.entries(evaluators)) {
+    validateEvaluatorName(name)
+    if (typeof evaluator !== 'function') throw new TypeError(`${kind} evaluator '${name}' must be a function`)
+    normalized.push([name, evaluator])
+  }
+  return normalized
+}
+
+/**
+ * @param {unknown} value
+ * @returns {string}
+ */
+function inferMetricType (value) {
+  if (typeof value === 'boolean') return 'boolean'
+  if (typeof value === 'number' && Number.isFinite(value)) return 'score'
+  if (typeof value === 'string') return 'categorical'
+  return 'json'
+}
+
+/**
+ * @param {unknown} value
+ * @param {WeakSet<object>} seen
+ * @returns {NormalizedJsonValue}
+ */
+function normalizeJsonValue (value, seen) {
+  if (value === null) return null
+
+  const valueType = typeof value
+  if (valueType === 'string' || valueType === 'boolean') return value
+  if (valueType === 'number') return Number.isFinite(value) ? value : String(value)
+  if (valueType === 'bigint') return value.toString()
+  if (valueType === 'undefined') return null
+  if (valueType === 'symbol') return value.toString()
+  if (valueType === 'function') return { type: 'function', name: value.name || '<anonymous>' }
+
+  if (value instanceof Date) return value.toISOString()
+  if (value instanceof Error) {
+    return { type: value.name, message: value.message, stack: value.stack ?? '' }
+  }
+
+  if (seen.has(value)) return '[Circular]'
+  seen.add(value)
+
+  if (Array.isArray(value)) {
+    const normalized = new Array(value.length)
+    for (let i = 0; i < value.length; i++) normalized[i] = normalizeJsonValue(value[i], seen)
+    seen.delete(value)
+    return normalized
+  }
+
+  const normalized = {}
+  for (const [key, nestedValue] of Object.entries(value)) {
+    normalized[key] = normalizeJsonValue(nestedValue, seen)
+  }
+  seen.delete(value)
+  return normalized
+}
+
+/**
+ * @param {unknown} value
+ * @returns {{ [key: string]: NormalizedJsonValue }}
+ */
+function normalizeJsonMetricValue (value) {
+  const normalized = normalizeJsonValue(value, new WeakSet())
+  if (normalized !== null && typeof normalized === 'object' && !Array.isArray(normalized)) return normalized
+  return { value: normalized }
+}
+
+/**
+ * @param {unknown} value
+ * @returns {string}
+ */
+function stringify (value) {
+  if (value == null) return ''
+  if (typeof value === 'string') return value.toLowerCase()
+  if (typeof value === 'object') {
+    try {
+      const stringified = JSON.stringify(value)
+      if (stringified !== undefined) return stringified.toLowerCase()
+    } catch {}
+  }
+  return String(value).toLowerCase()
+}
+
+/**
+ * @param {Record<string, unknown> | undefined} userTags
+ * @param {Record<string, unknown>} autoTags
+ * @returns {string[]}
+ */
+function buildTags (userTags, autoTags) {
+  const tags = new Map()
+  if ((userTags) != null) {
+    for (const [key, value] of Object.entries(userTags)) {
+      tags.set(key, `${key}:${value}`)
+    }
+  }
+  for (const [key, value] of Object.entries(autoTags)) {
+    if (value !== undefined && value !== null && value !== '') tags.set(key, `${key}:${value}`)
+  }
+  return [...tags.values()]
+}
+
+/**
+ * @param {Record<string, unknown> | undefined} userTags
+ * @param {Record<string, unknown>} autoTags
+ * @returns {Record<string, unknown>}
+ */
+function buildExperimentTagObject (userTags, autoTags) {
+  return userTags ? { ...userTags, ...autoTags } : { ...autoTags }
+}
+
+/**
+ * @param {number} ms
+ * @returns {Promise<void>}
+ */
+function sleep (ms) {
+  if (ms <= 0) return Promise.resolve()
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+/**
+ * @param {Record<string, unknown> | undefined} recordMetadata
+ * @param {Record<string, unknown>} config
+ * @returns {Record<string, unknown>}
+ */
+function buildSpanMetadata (recordMetadata, config) {
+  return recordMetadata
+    ? { ...recordMetadata, experiment_config: config }
+    : { experiment_config: config }
+}
+
+module.exports = {
+  buildExperimentTagObject,
+  buildSpanMetadata,
+  buildTags,
+  hasEntries,
+  inferMetricType,
+  normalizeEvaluators,
+  normalizeJsonMetricValue,
+  sleep,
+  stringify,
+  validateEvaluatorName,
+}

package/packages/dd-trace/src/llmobs/noop.js

@@ -80,6 +80,8 @@ class NoopLLMObs {
 
   submitEvaluation (llmobsSpanContext, options) {}
 
+  submitFeedback (options) {}
+
   flush () {}
 
   registerProcessor (processor) {}

package/packages/dd-trace/src/llmobs/plugins/ai/ddTelemetry.js

@@ -118,7 +118,7 @@ class DdTelemetryPlugin extends BaseLLMObsPlugin {
    * @returns {string | undefined}
    */
   findToolName (toolName, toolDescription) {
-    if (Number.isNaN(Number.parseInt(toolName))) return toolName
+    if (Number.isNaN(Number.parseInt(toolName, 10))) return toolName
 
     for (const availableTool of this.#availableTools) {
       const description = availableTool.description

package/packages/dd-trace/src/llmobs/plugins/ai/util.js

@@ -322,7 +322,7 @@ function getToolNameFromTags (tags) {
   const toolName = tags['ai.toolCall.name']
   if (!toolName) return null
 
-  const parsedToolName = Number.parseInt(toolName)
+  const parsedToolName = Number.parseInt(toolName, 10)
   if (!Number.isNaN(parsedToolName)) return null
 
   return toolName

package/packages/dd-trace/src/llmobs/plugins/anthropic/index.js

@@ -93,7 +93,7 @@ class AnthropicLLMObsPlugin extends LLMObsPlugin {
 
             const { usage } = chunk
             if (usage) {
-              const responseUsage = response.usage ?? (response.usage = { input_tokens: 0, output_tokens: 0 })
+              const responseUsage = (response.usage ??= { input_tokens: 0, output_tokens: 0 })
               responseUsage.output_tokens = usage.output_tokens
 
               const cacheCreationTokens = usage.cache_creation_input_tokens

package/packages/dd-trace/src/llmobs/plugins/bedrockruntime.js

@@ -72,10 +72,10 @@ class BedrockRuntimeLLMObsPlugin extends BaseLLMObsPlugin {
       const cacheWriteTokenCount = headers['x-amzn-bedrock-cache-write-input-token-count']
 
       pendingTokenHeaders.set(requestId, {
-        inputTokensFromHeaders: inputTokenCount && Number.parseInt(inputTokenCount),
-        outputTokensFromHeaders: outputTokenCount && Number.parseInt(outputTokenCount),
-        cacheReadTokensFromHeaders: cacheReadTokenCount && Number.parseInt(cacheReadTokenCount),
-        cacheWriteTokensFromHeaders: cacheWriteTokenCount && Number.parseInt(cacheWriteTokenCount),
+        inputTokensFromHeaders: inputTokenCount && Number.parseInt(inputTokenCount, 10),
+        outputTokensFromHeaders: outputTokenCount && Number.parseInt(outputTokenCount, 10),
+        cacheReadTokensFromHeaders: cacheReadTokenCount && Number.parseInt(cacheReadTokenCount, 10),
+        cacheWriteTokensFromHeaders: cacheWriteTokenCount && Number.parseInt(cacheWriteTokenCount, 10),
       })
     })
 
@@ -138,7 +138,7 @@ class BedrockRuntimeLLMObsPlugin extends BaseLLMObsPlugin {
   #tagCommon ({ span, requestParams, textAndResponseReason, tokensFromHeaders }) {
     this._tagger.tagMetadata(span, {
       temperature: Number.parseFloat(requestParams.temperature) || 0,
-      max_tokens: Number.parseInt(requestParams.maxTokens) || 0,
+      max_tokens: Number.parseInt(requestParams.maxTokens, 10) || 0,
     })
     this._tagger.tagLLMIO(span, requestParams.prompt, textAndResponseReason.messages)
     this._tagger.tagMetrics(span, extractTokens({

package/packages/dd-trace/src/llmobs/plugins/claude-agent-sdk/index.js

@@ -8,7 +8,7 @@ const { splitModel } = require('../../../../../datadog-plugin-claude-agent-sdk/s
 const subagentToolIds = new Set()
 
 function normalizeToolOutputString (raw) {
-  const footerIndex = raw.search(/\s*agentId: [^\s]+ \(use SendMessage\b/)
+  const footerIndex = raw.search(/agentId: \S+ \(use SendMessage\b/)
   if (footerIndex === -1) return raw
 
   return raw.slice(0, footerIndex).trimEnd()

package/packages/dd-trace/src/llmobs/plugins/genai/index.js

@@ -24,7 +24,7 @@ class GenAiLLMObsPlugin extends LLMObsPlugin {
     // Subscribe to streaming chunk events
     this.addSub('apm:google:genai:request:chunk', ({ ctx, chunk, done }) => {
       ctx.isStreaming = true
-      ctx.chunks = ctx.chunks || []
+      ctx.chunks ||= []
 
       if (chunk) ctx.chunks.push(chunk)
       if (!done) return

package/packages/dd-trace/src/llmobs/plugins/langchain/handlers/index.js

@@ -36,7 +36,7 @@ class LangChainLLMObsHandler {
     const runIdBase = runId ? runId.split('-').slice(0, -1).join('-') : ''
 
     const responseMetadata = message.response_metadata || {}
-    usage = usage || responseMetadata.usage || responseMetadata.tokenUsage || {}
+    usage ||= responseMetadata.usage || responseMetadata.tokenUsage || {}
 
     const inputTokens = usage.promptTokens || usage.inputTokens || usage.prompt_tokens || usage.input_tokens || 0
     const outputTokens =

package/packages/dd-trace/src/llmobs/plugins/langchain/index.js

@@ -113,7 +113,7 @@ class BaseLangChainLLMObsPlugin extends LLMObsPlugin {
     }
 
     if (maxTokens) {
-      metadata.maxTokens = Number.parseInt(maxTokens)
+      metadata.maxTokens = Number.parseInt(maxTokens, 10)
     }
 
     this._tagger.tagMetadata(span, metadata)