dd-trace 6.8.0 → 6.10.0

package/packages/dd-trace/src/llmobs/constants/tags.js

@@ -16,6 +16,11 @@ module.exports = {
   PROPAGATED_ML_APP_KEY: '_dd.p.llmobs_ml_app',
   PROPAGATED_SESSION_ID_KEY: '_dd.p.llmobs_sid',
   PARENT_ID_KEY: '_ml_obs.llmobs_parent_id',
+
+  PARENT_AGENT_NAME: '_ml_obs.parent_agent_name',
+  PARENT_AGENT_SPAN_ID: '_ml_obs.parent_agent_span_id',
+  PROPAGATED_PARENT_AGENT_ID_KEY: '_dd.p.llmobs_pagent_span_id',
+  PROPAGATED_PARENT_AGENT_NAME_KEY: '_dd.p.llmobs_pagent_name',
   PROPAGATED_SAMPLE_RATE_KEY: '_dd.p.llmobs_sr',
   PROPAGATED_SAMPLING_DECISION_KEY: '_dd.p.llmobs_sd',
   SAMPLE_RATE: '_ml_obs.sample_rate',

package/packages/dd-trace/src/llmobs/eval-metric.js

@@ -0,0 +1,249 @@
+'use strict'
+
+const tracerVersion = require('../../../../package.json').version
+
+/**
+ * Kind of event submitted through the evaluation metrics intake. Both kinds share the
+ * writer, the endpoint and most of their validation, and are told apart on the wire by
+ * the `event_kind` field.
+ * @typedef {'evaluation' | 'feedback'} MetricEventKind
+ */
+
+/**
+ * Tag value the intake accepts: a string, a nullish value, or anything the SDK can coerce
+ * through `toString()`.
+ * @typedef {string | null | undefined | { toString?: unknown }} MetricTagValue
+ */
+
+/** @type {string[]} */
+const EVALUATION_METRIC_TYPES = ['categorical', 'score', 'boolean', 'json']
+
+/**
+ * End-user feedback additionally accepts free-form text values.
+ * @type {string[]}
+ */
+const FEEDBACK_METRIC_TYPES = [...EVALUATION_METRIC_TYPES, 'text']
+
+/** @type {Record<MetricEventKind, string[]>} */
+const METRIC_TYPES = {
+  evaluation: EVALUATION_METRIC_TYPES,
+  feedback: FEEDBACK_METRIC_TYPES,
+}
+
+/**
+ * Feedback target option accepted by the SDK, mapped to the payload key it is sent as. `span` and
+ * `spanId` are two ways of naming the same target.
+ * @type {Record<string, string>}
+ */
+const FEEDBACK_TARGET_KEYS = {
+  span: 'span_id',
+  spanId: 'span_id',
+  traceId: 'trace_id',
+  sessionId: 'session_id',
+  feedbackJoinKey: 'feedback_join_key',
+}
+
+// The validation is shared but the user-facing wording is per-kind. Keeping the terms in one
+// table avoids threading three near-identical strings through every validator.
+/** @type {Record<MetricEventKind, { metric: string, metrics: string, data: string }>} */
+const TERMS = {
+  evaluation: {
+    metric: 'evaluation metric',
+    metrics: 'evaluation metrics',
+    data: 'Evaluation metric data',
+  },
+  feedback: {
+    metric: 'feedback metric',
+    metrics: 'feedback metrics',
+    data: 'Feedback data',
+  },
+}
+
+/**
+ * @param {string[]} metricTypes
+ * @returns {string} the quoted list, e.g. `"categorical", "score" or "json"`
+ */
+function formatMetricTypes (metricTypes) {
+  const quoted = metricTypes.map(metricType => `"${metricType}"`)
+  return `${quoted.slice(0, -1).join(', ')} or ${quoted.at(-1)}`
+}
+
+// Built once at load time: only ever used to build an error message.
+/** @type {Record<MetricEventKind, string>} */
+const METRIC_TYPES_MESSAGE = {
+  evaluation: formatMetricTypes(EVALUATION_METRIC_TYPES),
+  feedback: formatMetricTypes(FEEDBACK_METRIC_TYPES),
+}
+
+/**
+ * Builds an error carrying the telemetry error tag the SDK reports for the failed submission.
+ * @param {string} errorTag - Telemetry `error_type` tag value.
+ * @param {string} message - User-facing message.
+ * @returns {Error} the tagged error, to be thrown by the caller.
+ */
+function taggedError (errorTag, message) {
+  const error = new Error(message)
+  // Same shape as the tagger's failures, which the SDK already reads back as `e.ddErrorTag`.
+  Object.defineProperty(error, 'ddErrorTag', { get () { return errorTag } })
+  return error
+}
+
+/**
+ * @param {MetricEventKind} kind
+ * @returns {Error} the tagged error, to be thrown by the caller.
+ */
+function invalidTagsError (kind) {
+  return taggedError('invalid_tags', `Failed to parse tags. Tags for ${TERMS[kind].metrics} must be strings`)
+}
+
+/**
+ * @param {unknown} timestampMs
+ * @param {MetricEventKind} kind
+ * @returns {void}
+ */
+function validateTimestamp (timestampMs, kind) {
+  if (typeof timestampMs !== 'number' || timestampMs < 0) {
+    throw taggedError(
+      'invalid_timestamp',
+      `timestampMs must be a non-negative integer. ${TERMS[kind].data} will not be sent`
+    )
+  }
+}
+
+/**
+ * Validates the label and returns it as sent on the wire.
+ * @param {unknown} label
+ * @param {MetricEventKind} kind
+ * @returns {unknown} the label, coerced to a string for feedback and left untouched for evaluations.
+ */
+function validateLabel (label, kind) {
+  if (!label) {
+    throw taggedError('invalid_metric_label', `label must be the specified name of the ${TERMS[kind].metric}`)
+  }
+
+  // Evaluations keep the looser check they shipped with: a dotted or non-string label reaches the
+  // intake as-is. Aligning them with feedback rejects calls that succeed today, so it is deferred.
+  if (kind === 'evaluation') return label
+
+  // A dot makes the label unusable as a facet key.
+  const labelValue = String(label)
+  if (labelValue.includes('.')) {
+    throw taggedError('invalid_label_value', 'label value must not contain a "."')
+  }
+
+  return labelValue
+}
+
+/**
+ * @param {string | undefined} metricType - Already lower-cased metric type.
+ * @param {MetricEventKind} kind
+ * @returns {void}
+ */
+function validateMetricType (metricType, kind) {
+  if (!metricType || !METRIC_TYPES[kind].includes(metricType)) {
+    throw taggedError('invalid_metric_type', `metricType must be one of ${METRIC_TYPES_MESSAGE[kind]}`)
+  }
+}
+
+/**
+ * @param {string | undefined} metricType - Metric type already validated for the submitted kind.
+ * @param {unknown} value
+ * @returns {void}
+ */
+function validateMetricValue (metricType, value) {
+  if (metricType === 'categorical' && typeof value !== 'string') {
+    throw taggedError('invalid_metric_value', 'value must be a string for a categorical metric.')
+  }
+  if (metricType === 'score' && typeof value !== 'number') {
+    throw taggedError('invalid_metric_value', 'value must be a number for a score metric.')
+  }
+  if (metricType === 'boolean' && typeof value !== 'boolean') {
+    throw taggedError('invalid_metric_value', 'value must be a boolean for a boolean metric')
+  }
+  if (metricType === 'json' && (typeof value !== 'object' || value == null || Array.isArray(value))) {
+    throw taggedError('invalid_metric_value', 'value must be a JSON object for a json metric')
+  }
+  if (metricType === 'text' && typeof value !== 'string') {
+    throw taggedError('invalid_metric_value', 'value must be a string for a text metric')
+  }
+}
+
+/**
+ * @param {unknown} assessment
+ * @returns {void}
+ */
+function validateAssessment (assessment) {
+  if (assessment != null && assessment !== 'pass' && assessment !== 'fail') {
+    throw taggedError('invalid_assessment', 'assessment must be pass or fail')
+  }
+}
+
+/**
+ * @param {unknown} reasoning
+ * @returns {void}
+ */
+function validateReasoning (reasoning) {
+  if (reasoning != null && typeof reasoning !== 'string') {
+    throw taggedError('invalid_reasoning', 'reasoning must be a string')
+  }
+}
+
+/**
+ * Serializes the user-provided tags into the `key:value` list expected by the intake, along
+ * with the tags every submission carries.
+ * @param {Record<string, MetricTagValue> | undefined} tags - User-provided tags, coerced when not strings.
+ * @param {string} mlApp - Resolved ML app name.
+ * @param {MetricEventKind} kind
+ * @param {boolean} [otelEnabled] - Adds `source:otel` so the backend waits for OTel span conversion.
+ * @returns {string[]} the serialized tag list.
+ */
+function buildMetricTags (tags, mlApp, kind, otelEnabled = false) {
+  // An array or a string would otherwise be walked by index and produce tags named after
+  // their offsets, which the intake would happily store.
+  if (tags != null && (typeof tags !== 'object' || Array.isArray(tags))) {
+    throw invalidTagsError(kind)
+  }
+
+  const metricTags = {
+    'ddtrace.version': tracerVersion,
+    ml_app: mlApp,
+  }
+
+  if (tags) {
+    for (const key of Object.keys(tags)) {
+      const tag = tags[key]
+      if (typeof tag === 'string') {
+        metricTags[key] = tag
+      } else if (tag == null) {
+        // A nullish value can be intentional, so it is serialized rather than rejected. Checked
+        // before the `toString` lookup below, which would throw a raw TypeError on it.
+        metricTags[key] = String(tag)
+      } else if (typeof tag.toString === 'function') {
+        metricTags[key] = tag.toString()
+      } else {
+        // Every other value carries a `toString`, either its own or the one inherited from
+        // `Object.prototype`, so this is only reached for a null-prototype object.
+        throw invalidTagsError(kind)
+      }
+    }
+  }
+
+  if (otelEnabled) {
+    metricTags.source = 'otel'
+  }
+
+  return Object.entries(metricTags).map(([key, value]) => `${key}:${value}`)
+}
+
+module.exports = {
+  EVALUATION_METRIC_TYPES,
+  FEEDBACK_METRIC_TYPES,
+  FEEDBACK_TARGET_KEYS,
+  buildMetricTags,
+  validateAssessment,
+  validateLabel,
+  validateMetricType,
+  validateMetricValue,
+  validateReasoning,
+  validateTimestamp,
+}

package/packages/dd-trace/src/llmobs/experiments/index.js

@@ -47,15 +47,17 @@ class Experiments {
       : (descriptionOrOptions ?? {})
     const dataset = new Dataset(this.#client, name, options.description ?? '')
     const recordIds = new Set()
-    for (const record of options.records ?? []) {
-      if (record.id !== undefined && (typeof record.id !== 'string' || record.id.length === 0)) {
-        throw new Error('record id must be a non-empty string')
-      }
-      if (record.id !== undefined) {
-        if (recordIds.has(record.id)) throw new Error(`Duplicate record id '${record.id}'`)
-        recordIds.add(record.id)
+    if ((options.records) != null) {
+      for (const record of options.records) {
+        if (record.id !== undefined && (typeof record.id !== 'string' || record.id.length === 0)) {
+          throw new Error('record id must be a non-empty string')
+        }
+        if (record.id !== undefined) {
+          if (recordIds.has(record.id)) throw new Error(`Duplicate record id '${record.id}'`)
+          recordIds.add(record.id)
+        }
+        dataset.addRecord(new DatasetRecord(record.inputData, record.expectedOutput, record.metadata, record.id))
       }
-      dataset.addRecord(new DatasetRecord(record.inputData, record.expectedOutput, record.metadata, record.id))
     }
     return dataset
   }
@@ -82,13 +84,16 @@ class Experiments {
             'GET',
             `${API_BASE_PATH}/${projectId}/datasets?filter[name]=${encodeURIComponent(name)}`
           )
-          for (const item of listed?.data ?? []) {
-            if (item?.attributes?.name === name) {
-              datasetId = String(item?.id ?? '')
-              description = String(item?.attributes?.description ?? '')
-              latestVersion = item?.attributes?.current_version ?? null
-              datasetVersion = version ?? latestVersion
-              break
+          const datasets = listed?.data
+          if (datasets) {
+            for (const item of datasets) {
+              if (item?.attributes?.name === name) {
+                datasetId = String(item?.id ?? '')
+                description = String(item?.attributes?.description ?? '')
+                latestVersion = item?.attributes?.current_version ?? null
+                datasetVersion = version ?? latestVersion
+                break
+              }
             }
           }
           if (datasetId === null) return false
@@ -107,16 +112,19 @@ class Experiments {
             'GET',
             `${API_BASE_PATH}/${projectId}/datasets/${datasetId}/records?${query.toString()}`
           )
-          for (const item of resp?.data ?? []) {
-            const attrs = item?.attributes ?? item
-            const recordId = String(item?.id ?? attrs?.id ?? '')
-            recs.push(new DatasetRecord(
-              attrs?.input ?? null,
-              attrs?.expected_output ?? null,
-              attrs?.metadata ?? {},
-              recordId === '' ? null : recordId
-            ))
-            ids.push(recordId)
+          const recordData = resp?.data
+          if (recordData) {
+            for (const item of recordData) {
+              const attrs = item?.attributes ?? item
+              const recordId = String(item?.id ?? attrs?.id ?? '')
+              recs.push(new DatasetRecord(
+                attrs?.input ?? null,
+                attrs?.expected_output ?? null,
+                attrs?.metadata ?? {},
+                recordId === '' ? null : recordId
+              ))
+              ids.push(recordId)
+            }
           }
           cursor = resp?.meta?.after ?? ''
           if (!cursor) break

package/packages/dd-trace/src/llmobs/experiments/util.js

@@ -166,8 +166,10 @@ function stringify (value) {
  */
 function buildTags (userTags, autoTags) {
   const tags = new Map()
-  for (const [key, value] of Object.entries(userTags ?? {})) {
-    tags.set(key, `${key}:${value}`)
+  if ((userTags) != null) {
+    for (const [key, value] of Object.entries(userTags)) {
+      tags.set(key, `${key}:${value}`)
+    }
   }
   for (const [key, value] of Object.entries(autoTags)) {
     if (value !== undefined && value !== null && value !== '') tags.set(key, `${key}:${value}`)

package/packages/dd-trace/src/llmobs/index.js

@@ -2,6 +2,7 @@
 
 const { channel } = require('dc-polyfill')
 
+const { readDatadogTags, writeDatadogTags } = require('../carrier')
 const log = require('../log')
 const { DD_MAJOR } = require('../../../../version')
 const startupLogs = require('../startup-log')
@@ -12,6 +13,8 @@ const {
   PROPAGATED_ML_APP_KEY,
   PROPAGATED_PARENT_ID_KEY,
   PROPAGATED_SESSION_ID_KEY,
+  PROPAGATED_PARENT_AGENT_ID_KEY,
+  PROPAGATED_PARENT_AGENT_NAME_KEY,
   SAMPLE_RATE,
   SAMPLING_DECISION,
   PROPAGATED_SAMPLE_RATE_KEY,
@@ -20,6 +23,7 @@ const {
   PROPAGATED_TRACE_ID_KEY,
 } = require('./constants/tags')
 const { storage } = require('./storage')
+const { agentNameWireSafe, appendOptionalPropagatedTag, resolveAgentAttribution, stripTagsetEntry } = require('./util')
 const telemetry = require('./telemetry')
 const LLMObsSpanProcessor = require('./span_processor')
 const LLMObsEvalMetricsWriter = require('./writers/evaluations')
@@ -147,9 +151,17 @@ function handleLLMObsInjection ({ carrier }) {
 
   if (!parentId && !mlApp && samplingDecision == null && !sessionId && !propagatedTraceId) return
 
+  // Propagate the nearest agent so spans in the downstream process attribute correctly. When the
+  // active span sits under a distributed agent, `resolveAgentAttribution` inherits the propagated
+  // id/name already on the parent's registry entry, so the chain survives multiple hops. Resolved
+  // after the bail-out above so we don't allocate when there is nothing to inject.
+  const { name: parentAgentName, spanId: parentAgentSpanId } = resolveAgentAttribution(
+    mlObsSpanTags, parent
+  )
+
   // `_injectTags` only writes `x-datadog-tags` when the trace has `_dd.p.*`
   // tags, so it may be undefined here — coalesce before appending.
-  const existing = carrier['x-datadog-tags']
+  const existing = readDatadogTags(carrier)
   let tags = existing || ''
   if (parentId) tags += `${tags ? ',' : ''}${PROPAGATED_PARENT_ID_KEY}=${parentId}`
   if (mlApp) tags += `${tags ? ',' : ''}${PROPAGATED_ML_APP_KEY}=${mlApp}`
@@ -157,7 +169,27 @@ function handleLLMObsInjection ({ carrier }) {
   if (sampleRate != null) tags += `${tags ? ',' : ''}${PROPAGATED_SAMPLE_RATE_KEY}=${sampleRate}`
   if (samplingDecision != null) tags += `${tags ? ',' : ''}${PROPAGATED_SAMPLING_DECISION_KEY}=${samplingDecision}`
   if (propagatedTraceId != null) tags += `${tags ? ',' : ''}${PROPAGATED_TRACE_ID_KEY}=${propagatedTraceId}`
-  if (tags !== existing) carrier['x-datadog-tags'] = tags
+  // When a local agent attribution is resolved, strip any stale upstream pagent entries that
+  // `_injectTags` may have already written into the carrier (it propagates all `_dd.p.*` from
+  // `_trace.tags`). This ensures the downstream sees a consistent id-only or id+name pair
+  // rather than a mix from different hops. The id is always digit-safe; an unsafe name is wiped.
+  if (parentAgentSpanId) {
+    tags = stripTagsetEntry(tags, PROPAGATED_PARENT_AGENT_ID_KEY)
+    tags = stripTagsetEntry(tags, PROPAGATED_PARENT_AGENT_NAME_KEY)
+  }
+  const maxLength = globalTracerConfig.DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH
+  const tagsWithId = appendOptionalPropagatedTag(
+    tags, PROPAGATED_PARENT_AGENT_ID_KEY, parentAgentSpanId, null, maxLength
+  )
+  // Only append the name when the id fit: a name without an id is unresolvable by the backend.
+  if (tagsWithId === tags) {
+    tags = tagsWithId
+  } else {
+    tags = appendOptionalPropagatedTag(
+      tagsWithId, PROPAGATED_PARENT_AGENT_NAME_KEY, parentAgentName, agentNameWireSafe, maxLength
+    )
+  }
+  if (tags !== existing) writeDatadogTags(carrier, tags)
 }
 
 function handleFlush () {

package/packages/dd-trace/src/llmobs/noop.js

@@ -80,6 +80,8 @@ class NoopLLMObs {
 
   submitEvaluation (llmobsSpanContext, options) {}
 
+  submitFeedback (options) {}
+
   flush () {}
 
   registerProcessor (processor) {}

package/packages/dd-trace/src/llmobs/plugins/ai/ddTelemetry.js

@@ -118,7 +118,7 @@ class DdTelemetryPlugin extends BaseLLMObsPlugin {
    * @returns {string | undefined}
    */
   findToolName (toolName, toolDescription) {
-    if (Number.isNaN(Number.parseInt(toolName))) return toolName
+    if (Number.isNaN(Number.parseInt(toolName, 10))) return toolName
 
     for (const availableTool of this.#availableTools) {
       const description = availableTool.description

package/packages/dd-trace/src/llmobs/plugins/ai/util.js

@@ -322,7 +322,7 @@ function getToolNameFromTags (tags) {
   const toolName = tags['ai.toolCall.name']
   if (!toolName) return null
 
-  const parsedToolName = Number.parseInt(toolName)
+  const parsedToolName = Number.parseInt(toolName, 10)
   if (!Number.isNaN(parsedToolName)) return null
 
   return toolName

package/packages/dd-trace/src/llmobs/plugins/bedrockruntime.js

@@ -72,10 +72,10 @@ class BedrockRuntimeLLMObsPlugin extends BaseLLMObsPlugin {
       const cacheWriteTokenCount = headers['x-amzn-bedrock-cache-write-input-token-count']
 
       pendingTokenHeaders.set(requestId, {
-        inputTokensFromHeaders: inputTokenCount && Number.parseInt(inputTokenCount),
-        outputTokensFromHeaders: outputTokenCount && Number.parseInt(outputTokenCount),
-        cacheReadTokensFromHeaders: cacheReadTokenCount && Number.parseInt(cacheReadTokenCount),
-        cacheWriteTokensFromHeaders: cacheWriteTokenCount && Number.parseInt(cacheWriteTokenCount),
+        inputTokensFromHeaders: inputTokenCount && Number.parseInt(inputTokenCount, 10),
+        outputTokensFromHeaders: outputTokenCount && Number.parseInt(outputTokenCount, 10),
+        cacheReadTokensFromHeaders: cacheReadTokenCount && Number.parseInt(cacheReadTokenCount, 10),
+        cacheWriteTokensFromHeaders: cacheWriteTokenCount && Number.parseInt(cacheWriteTokenCount, 10),
       })
     })
 
@@ -138,7 +138,7 @@ class BedrockRuntimeLLMObsPlugin extends BaseLLMObsPlugin {
   #tagCommon ({ span, requestParams, textAndResponseReason, tokensFromHeaders }) {
     this._tagger.tagMetadata(span, {
       temperature: Number.parseFloat(requestParams.temperature) || 0,
-      max_tokens: Number.parseInt(requestParams.maxTokens) || 0,
+      max_tokens: Number.parseInt(requestParams.maxTokens, 10) || 0,
     })
     this._tagger.tagLLMIO(span, requestParams.prompt, textAndResponseReason.messages)
     this._tagger.tagMetrics(span, extractTokens({

package/packages/dd-trace/src/llmobs/plugins/langchain/index.js

@@ -113,7 +113,7 @@ class BaseLangChainLLMObsPlugin extends LLMObsPlugin {
     }
 
     if (maxTokens) {
-      metadata.maxTokens = Number.parseInt(maxTokens)
+      metadata.maxTokens = Number.parseInt(maxTokens, 10)
     }
 
     this._tagger.tagMetadata(span, metadata)