dd-trace 6.18.0 → 6.19.0

package/packages/dd-trace/src/llmobs/plugins/base.js

@@ -1,32 +1,89 @@
 'use strict'
 
 const log = require('../../log')
+const { PROPAGATED_SESSION_ID_KEY } = require('../constants/tags')
+const { MODEL_BACKED_SPAN_KINDS, setGenAiApmTags, updateGenAiApmTags } = require('../gen-ai-tags')
 const { storage: llmobsStorage } = require('../storage')
 const telemetry = require('../telemetry')
 
 const TracingPlugin = require('../../plugins/tracing')
 const LLMObsTagger = require('../tagger')
 
+/**
+ * @typedef {object} LLMObsSpanRegisterOptions
+ * @property {string} kind LLMObs span kind
+ * @property {string} [name]
+ * @property {string} [modelName]
+ * @property {string} [modelProvider]
+ * @property {string} [mlApp]
+ * @property {string} [sessionId]
+ */
+
 class LLMObsPlugin extends TracingPlugin {
+  /**
+   * Whether this integration emits the `gen_ai.*` APM attributes while LLM Observability is off.
+   * An integration sets this to `false` when staying subscribed would cost more than the tags are
+   * worth, and then behaves as it did before those attributes existed: disabled outright.
+   */
+  static emitsGenAiApmTags = true
+
   constructor (...args) {
     super(...args)
 
     this._tagger = new LLMObsTagger(this._tracerConfig, true)
   }
 
+  /**
+   * Whether the LLMObs layer is active. When it is not, the plugin stays subscribed but only
+   * emits the `gen_ai.*` APM attributes.
+   *
+   */
+  get _llmobsEnabled () {
+    return this._tracerConfig.llmobs.DD_LLMOBS_ENABLED
+  }
+
+  /**
+   * The mode one operation runs in, latched on first read. `llmobs.enable()` and `llmobs.disable()`
+   * flip the flag while operations are in flight, and an operation that switches track halfway
+   * reports neither: the tagger rejects a span the start never registered, and the reduced path
+   * has no start tags for its end hook to update. Every hook for a given operation reads it here
+   * so they all take the branch its start took.
+   *
+   * @param {object} ctx
+   */
+  _llmobsEnabledFor (ctx) {
+    ctx.llmobsEnabled ??= this._llmobsEnabled
+    return ctx.llmobsEnabled
+  }
+
   setLLMObsTags (ctx) {
     throw new Error('setLLMObsTags must be implemented by the subclass')
   }
 
+  /**
+   * The `gen_ai.*` values an integration can only resolve once the operation finished, such as
+   * token usage or a session id the response carries. Only used while LLMObs is disabled; the
+   * LLMObs layer reads them off the span event instead. Fields left out keep their start value.
+   *
+   * @param {object} ctx
+   * @param {string} spanKind LLMObs span kind resolved at span start
+   * @returns {import('../gen-ai-tags').GenAiApmTags | void}
+   */
+  getGenAiApmEndTags (ctx, spanKind) {}
+
+  /**
+   * @param {object} ctx
+   * @returns {LLMObsSpanRegisterOptions | undefined}
+   */
   getLLMObsSpanRegisterOptions (ctx) {
     throw new Error('getLLMObsSPanRegisterOptions must be implemented by the subclass')
   }
 
   start (ctx) {
-    // even though llmobs span events won't be enqueued if llmobs is disabled
-    // we should avoid doing any computations here (these listeners aren't disabled)
-    const enabled = this._tracerConfig.llmobs.DD_LLMOBS_ENABLED
-    if (!enabled) return
+    if (!this._llmobsEnabledFor(ctx)) {
+      this.#setGenAiApmTagsFromRegisterOptions(ctx)
+      return
+    }
 
     const parentStore = llmobsStorage.getStore()
     const apmStore = ctx.currentStore
@@ -52,8 +109,7 @@ class LLMObsPlugin extends TracingPlugin {
   }
 
   end (ctx) {
-    const enabled = this._tracerConfig.llmobs.DD_LLMOBS_ENABLED
-    if (!enabled) return
+    if (!this._llmobsEnabledFor(ctx)) return
 
     // only attempt to restore the context if the current span was an LLMObs span
     const apmStore = ctx.currentStore
@@ -65,10 +121,10 @@ class LLMObsPlugin extends TracingPlugin {
   }
 
   asyncEnd (ctx) {
-    // even though llmobs span events won't be enqueued if llmobs is disabled
-    // we should avoid doing any computations here (these listeners aren't disabled)
-    const enabled = this._tracerConfig.llmobs.DD_LLMOBS_ENABLED
-    if (!enabled) return
+    if (!this._llmobsEnabledFor(ctx)) {
+      this.#setGenAiApmEndTags(ctx)
+      return
+    }
 
     const apmStore = ctx.currentStore
     const span = apmStore?.span
@@ -83,12 +139,95 @@ class LLMObsPlugin extends TracingPlugin {
     this.setLLMObsTags(ctx)
   }
 
+  /**
+   * Resolves the LLMObs annotations the `gen_ai.*` APM attributes need from the span register
+   * options, which every integration already builds for the LLMObs layer.
+   *
+   * @param {object} ctx
+   */
+  #setGenAiApmTagsFromRegisterOptions (ctx) {
+    const span = ctx.currentStore?.span
+    if (!span) return
+
+    try {
+      const registerOptions = this.getLLMObsSpanRegisterOptions(ctx)
+      if (!registerOptions?.kind) return
+
+      // `asyncEnd` needs these back: the kind to gate the usage metrics, and the model in case an
+      // integration promotes the kind to one that must always report a model
+      ctx.genAiApmStartTags = {
+        spanKind: registerOptions.kind,
+        modelName: registerOptions.modelName,
+        modelProvider: registerOptions.modelProvider,
+        mlApp: registerOptions.mlApp,
+        sessionId: registerOptions.sessionId,
+      }
+
+      this._setGenAiApmTags(span, ctx.genAiApmStartTags)
+    } catch (e) {
+      log.debug('Failed to set gen_ai APM tags for %s:', this.constructor.name, e.message)
+    }
+  }
+
+  /**
+   * @param {object} ctx
+   */
+  #setGenAiApmEndTags (ctx) {
+    const span = ctx.currentStore?.span
+    const startTags = ctx.genAiApmStartTags
+    if (!span || !startTags) return
+
+    const { spanKind } = startTags
+
+    try {
+      const endTags = this.getGenAiApmEndTags(ctx, spanKind)
+      if (!endTags) return
+
+      // An integration may correct the kind, the way the tagger's `changeKind` does. A correction
+      // into a model-backed kind has to go back through `setGenAiApmTags`, which applies the model
+      // and provider defaults a model-backed span always reports; an update alone would leave the
+      // span claiming a kind the enabled path could never emit without a model.
+      const promotedToModelBacked = endTags.spanKind &&
+        endTags.spanKind !== spanKind &&
+        MODEL_BACKED_SPAN_KINDS.has(endTags.spanKind)
+
+      if (promotedToModelBacked) {
+        this._setGenAiApmTags(span, { ...startTags, ...endTags })
+      } else {
+        updateGenAiApmTags(span, { spanKind, ...endTags })
+      }
+    } catch (e) {
+      log.debug('Failed to set gen_ai APM end tags for %s:', this.constructor.name, e.message)
+    }
+  }
+
+  /**
+   * Writes the `gen_ai.*` APM attributes.
+   *
+   * No `gen_ai.application.name`: ml_app is an LLM Observability concept, and with LLMObs off the
+   * only value left to report is the service name the span already carries. dd-trace-py's reduced
+   * path leaves it out for the same reason.
+   *
+   * @param {import('../../opentracing/span')} span
+   * @param {import('../gen-ai-tags').GenAiApmTags} tags
+   */
+  _setGenAiApmTags (span, tags) {
+    // the in-process session default is written by the tagger, which never runs on this path, so
+    // an inherited session can only have come from an upstream service
+    const propagatedSessionId = span.context()._trace.tags[PROPAGATED_SESSION_ID_KEY]
+
+    setGenAiApmTags(span, { ...tags, sessionId: tags.sessionId || propagatedSessionId })
+  }
+
   configure (config) {
-    // we do not want to enable any LLMObs plugins if it is disabled on the tracer, or if the
-    // integration opted out via `tracer.use(<name>, { llmobs: false })`. Opting out only disables
-    // the LLMObs layer: the integration keeps emitting APM spans and propagating trace context.
-    const llmobsEnabled = this._tracerConfig.llmobs.DD_LLMOBS_ENABLED
-    if (llmobsEnabled === false || config?.llmobs === false) {
+    // an integration opt-out via `tracer.use(<name>, { llmobs: false })` disables the LLMObs layer
+    // entirely. When only LLMObs itself is disabled we stay subscribed: the handlers then emit the
+    // `gen_ai.*` APM attributes and skip the LLMObs payload, unless the integration opted out of
+    // those too.
+    const disabled = config?.llmobs === false ||
+      (!this._llmobsEnabled && !this.constructor.emitsGenAiApmTags)
+
+    if (disabled) {
       config = typeof config === 'boolean' ? false : { ...config, enabled: false } // override to false
     }
     super.configure(config)

package/packages/dd-trace/src/llmobs/plugins/bedrockruntime.js

@@ -1,10 +1,14 @@
 'use strict'
 
 const { storage } = require('../../../../datadog-core')
+const log = require('../../log')
 const telemetry = require('../telemetry')
+const { safeJsonParse } = require('../util')
 const {
+  buildUsage,
   extractRequestParams,
   extractTextAndResponseReason,
+  mergeStreamedUsage,
   parseModelId,
   extractTextAndResponseReasonFromStream,
   extractConverseToolDefinitions,
@@ -52,12 +56,45 @@ class BedrockRuntimeLLMObsPlugin extends BaseLLMObsPlugin {
       // avoids instrumenting other non supported runtime operations
       if (!ENABLED_OPERATIONS.has(operation)) return
 
-      const { modelProvider, modelName } = parseModelId(request.params.modelId)
+      // the SDK rejects a request with no model id, and the parser assumes a string
+      const modelId = request.params?.modelId
+      if (typeof modelId !== 'string') return
+
+      const { modelProvider, modelName } = parseModelId(modelId)
 
       // avoids instrumenting non llm type
       if (modelName.includes('embed')) return
 
       const span = ctx.currentStore?.span
+      if (!span) return
+
+      if (!this._llmobsEnabledFor(ctx)) {
+        // no LLMObs payload to build, so the usage comes from the response headers and, where
+        // those are absent, from whatever reported it
+        let usage
+        if (CONVERSE_OPERATIONS.has(operation)) {
+          // a non-streamed Converse puts it on the response, a streamed one on a metadata event
+          usage = buildUsage(response.usage) ?? ctx.streamedUsage
+        } else if (operation.toLowerCase().includes('stream')) {
+          // every streamed frame was folded into the running totals as it arrived
+          usage = ctx.streamedUsage
+        } else {
+          // headers can report some counts and the body others, so both are read and
+          // `extractTokens` merges them field by field
+          usage = responseBodyUsage(response, modelProvider, modelName)
+        }
+
+        this._setGenAiApmTags(span, {
+          spanKind: 'llm',
+          modelName: modelId.toLowerCase(),
+          modelProvider: 'amazon_bedrock',
+          // a count no source reported comes back undefined and is left off the span: reporting
+          // zeros for every metric would be worse than reporting none
+          metrics: extractTokens({ tokensFromHeaders, usage: usage ?? {} }),
+        })
+        return
+      }
+
       this.setLLMObsTags({ ctx, request, span, response, modelProvider, modelName, tokensFromHeaders })
     })
 
@@ -71,6 +108,10 @@ class BedrockRuntimeLLMObsPlugin extends BaseLLMObsPlugin {
       const cacheReadTokenCount = headers['x-amzn-bedrock-cache-read-input-token-count']
       const cacheWriteTokenCount = headers['x-amzn-bedrock-cache-write-input-token-count']
 
+      // Responses that report no counts at all, error responses included, would otherwise cache a
+      // record of undefined fields that reads as a measurement of zero.
+      if (!inputTokenCount && !outputTokenCount && !cacheReadTokenCount && !cacheWriteTokenCount) return
+
       pendingTokenHeaders.set(requestId, {
         inputTokensFromHeaders: inputTokenCount && Number.parseInt(inputTokenCount, 10),
         outputTokensFromHeaders: outputTokenCount && Number.parseInt(outputTokenCount, 10),
@@ -80,6 +121,13 @@ class BedrockRuntimeLLMObsPlugin extends BaseLLMObsPlugin {
     })
 
     this.addSub('apm:aws:response:streamed-chunk:bedrockruntime', ({ ctx, chunk }) => {
+      if (!this._llmobsEnabledFor(ctx)) {
+        // only the token counts are needed, for the `gen_ai.usage.*` metrics; the generated
+        // content is left to the LLMObs path, so nothing is retained past the running totals
+        ctx.streamedUsage = mergeChunkUsage(ctx, chunk)
+        return
+      }
+
       if (!ctx.chunks) ctx.chunks = []
 
       if (chunk) ctx.chunks.push(chunk)
@@ -141,10 +189,10 @@ class BedrockRuntimeLLMObsPlugin extends BaseLLMObsPlugin {
       max_tokens: Number.parseInt(requestParams.maxTokens, 10) || 0,
     })
     this._tagger.tagLLMIO(span, requestParams.prompt, textAndResponseReason.messages)
-    this._tagger.tagMetrics(span, extractTokens({
+    this._tagger.tagMetrics(span, zeroFilled(extractTokens({
       tokensFromHeaders,
       usage: textAndResponseReason.usage,
-    }))
+    })))
   }
 }
 
@@ -159,9 +207,74 @@ function consumeTokenHeaders (requestId) {
 }
 
 /**
- * Combine response-body usage with header-derived counts, preferring the body.
+ * Fold one streamed frame's token counts into the totals on `ctx`. Converse reports them on a
+ * metadata event; `invokeModel` reports them in the frame body, in a shape that varies by
+ * provider, so the body is read through the same table the LLMObs path uses.
+ *
+ * @param {object} ctx
+ * @param {object} [chunk]
+ * @returns {import('../../../../datadog-plugin-aws-sdk/src/services/bedrockruntime/utils')
+ *   .StreamedUsage | undefined}
+ */
+function mergeChunkUsage (ctx, chunk) {
+  const metadataUsage = chunk?.metadata?.usage
+  if (metadataUsage) return buildUsage(metadataUsage) ?? ctx.streamedUsage
+
+  const bytes = chunk?.chunk?.bytes
+  if (!ArrayBuffer.isView(bytes)) return ctx.streamedUsage
+
+  // a view, not a copy: this runs on every frame of every streamed response
+  const text = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString('utf8')
+  const body = safeJsonParse(text, null)
+  // a frame the model filled with generated text rather than JSON must not reach the application
+  if (typeof body !== 'object' || body === null) return ctx.streamedUsage
+
+  return mergeStreamedUsage(ctx.streamedUsage, body, streamModelProvider(ctx))
+}
+
+/**
+ * Token usage a non-streamed `invokeModel` reports in its own response body, which several
+ * providers carry and the headers do not always correlate. Read through the same extractor the
+ * LLMObs path uses, which parses this body on every request anyway.
+ *
+ * @param {{ body?: Uint8Array }} response
+ * @param {string} modelProvider
+ * @param {string} modelName
+ * @returns {Record<string, number | undefined> | undefined}
+ */
+function responseBodyUsage (response, modelProvider, modelName) {
+  if (!response?.body) return
+
+  try {
+    return extractTextAndResponseReason(response, modelProvider, modelName).usage
+  } catch (e) {
+    // the extractor parses the body itself; a malformed one must not disable the plugin
+    log.debug('Failed to read Bedrock response usage: %s', e.message)
+  }
+}
+
+/**
+ * The provider is fixed for the life of the stream, so it is parsed off the request once.
+ *
+ * @param {object} ctx
+ */
+function streamModelProvider (ctx) {
+  if (ctx.streamModelProvider === undefined) {
+    const modelId = (ctx.request ?? ctx.response?.request)?.params?.modelId
+    ctx.streamModelProvider = typeof modelId === 'string'
+      ? parseModelId(modelId).modelProvider.toUpperCase()
+      : ''
+  }
+
+  return ctx.streamModelProvider
+}
+
+/**
+ * Combine response-body usage with header-derived counts, preferring the body. A count no source
+ * reported stays undefined rather than becoming a zero that reads as a measurement.
  *
  * @param {{ tokensFromHeaders: HeaderTokens | undefined, usage: Record<string, number | undefined> }} options
+ * @returns {Record<string, number | undefined>}
  */
 function extractTokens ({ tokensFromHeaders, usage }) {
   const {
@@ -171,21 +284,60 @@ function extractTokens ({ tokensFromHeaders, usage }) {
     cacheWriteTokensFromHeaders,
   } = tokensFromHeaders ?? {}
 
-  const inputTokens = usage.inputTokens || inputTokensFromHeaders || 0
-  const outputTokens = usage.outputTokens || outputTokensFromHeaders || 0
-  const cacheReadTokens = usage.cacheReadTokens || cacheReadTokensFromHeaders || 0
-  const cacheWriteTokens = usage.cacheWriteTokens || cacheWriteTokensFromHeaders || 0
+  const inputTokens = resolveCount(usage.inputTokens, inputTokensFromHeaders)
+  const outputTokens = resolveCount(usage.outputTokens, outputTokensFromHeaders)
+  const cacheReadTokens = resolveCount(usage.cacheReadTokens, cacheReadTokensFromHeaders)
+  const cacheWriteTokens = resolveCount(usage.cacheWriteTokens, cacheWriteTokensFromHeaders)
 
   // adjust for the fact that bedrock input tokens only count non-cached tokens
-  const normalizedInputTokens = inputTokens + cacheReadTokens + cacheWriteTokens
+  const normalizedInputTokens = inputTokens === undefined &&
+    cacheReadTokens === undefined &&
+    cacheWriteTokens === undefined
+    ? undefined
+    : (inputTokens ?? 0) + (cacheReadTokens ?? 0) + (cacheWriteTokens ?? 0)
+
+  const totalTokens = normalizedInputTokens === undefined && outputTokens === undefined
+    ? undefined
+    : (normalizedInputTokens ?? 0) + (outputTokens ?? 0)
 
   return {
     inputTokens: normalizedInputTokens,
     outputTokens,
-    totalTokens: normalizedInputTokens + outputTokens,
+    totalTokens,
     cacheReadTokens,
     cacheWriteTokens,
   }
 }
 
+/**
+ * The body wins over the headers whenever it reported a count, a measured zero included, and a
+ * value neither reported as a number is left undefined: header counts are parsed from strings
+ * and can arrive empty.
+ *
+ * @param {unknown} fromBody
+ * @param {unknown} fromHeaders
+ * @returns {number | undefined}
+ */
+function resolveCount (fromBody, fromHeaders) {
+  const value = typeof fromBody === 'number' ? fromBody : fromHeaders
+  return typeof value === 'number' && !Number.isNaN(value) ? value : undefined
+}
+
+/**
+ * The LLMObs metrics contract reports an unmeasured count as zero, where the `gen_ai.*` APM
+ * attributes leave it off the span entirely.
+ *
+ * @param {Record<string, number | undefined>} tokens
+ * @returns {Record<string, number>}
+ */
+function zeroFilled (tokens) {
+  return {
+    inputTokens: tokens.inputTokens ?? 0,
+    outputTokens: tokens.outputTokens ?? 0,
+    totalTokens: tokens.totalTokens ?? 0,
+    cacheReadTokens: tokens.cacheReadTokens ?? 0,
+    cacheWriteTokens: tokens.cacheWriteTokens ?? 0,
+  }
+}
+
 module.exports = BedrockRuntimeLLMObsPlugin

package/packages/dd-trace/src/llmobs/plugins/claude-agent-sdk/index.js

@@ -5,6 +5,7 @@ const { storage: llmobsStorage } = require('../../storage')
 const { NAME, SESSION_ID } = require('../../constants/tags')
 const { splitModel } = require('../../../../../datadog-plugin-claude-agent-sdk/src/util')
 
+const SYSTEM_PROMPT_DYNAMIC_BOUNDARY = '__SYSTEM_PROMPT_DYNAMIC_BOUNDARY__'
 const subagentToolIds = new Set()
 
 function normalizeToolOutputString (raw) {
@@ -39,6 +40,27 @@ function getToolOutputText (raw) {
   return JSON.stringify(raw)
 }
 
+/**
+ * @param {object} [usage]
+ * @returns {Record<string, number> | undefined}
+ */
+function extractUsageMetrics (usage) {
+  if (!usage) return
+
+  const cacheWriteTokens = usage.cache_creation_input_tokens ?? 0
+  const cacheReadTokens = usage.cache_read_input_tokens ?? 0
+  const inputTokens = (usage.input_tokens ?? 0) + cacheWriteTokens + cacheReadTokens
+  const outputTokens = usage.output_tokens ?? 0
+
+  return {
+    input_tokens: inputTokens,
+    output_tokens: outputTokens,
+    cache_read_input_tokens: cacheReadTokens,
+    cache_write_input_tokens: cacheWriteTokens,
+    total_tokens: inputTokens + outputTokens,
+  }
+}
+
 function buildOutputMessages (chunks, llmStartIdx, llmEndIdx) {
   let thinking = ''
   let text = ''
@@ -85,6 +107,13 @@ class QueryLLMObsPlugin extends LLMObsPlugin {
     super.asyncEnd(ctx)
   }
 
+  /**
+   * @override
+   */
+  getGenAiApmEndTags (ctx) {
+    return { sessionId: ctx.session_id }
+  }
+
   setLLMObsTags (ctx) {
     const span = ctx.currentStore?.span
     if (!span) return
@@ -99,6 +128,11 @@ class QueryLLMObsPlugin extends LLMObsPlugin {
 
     if (cwd) metadata.cwd = cwd
     if (permissionMode) metadata.permissionMode = permissionMode
+    const systemPrompt = ctx.arguments?.[0]?.options?.systemPrompt
+    if (systemPrompt?.type === 'preset') {
+      if (typeof systemPrompt.preset === 'string') metadata.systemPromptPreset = systemPrompt.preset
+      if (typeof systemPrompt.append === 'string') metadata.systemPromptAppend = systemPrompt.append
+    }
 
     this._tagger.tagMetadata(span, metadata)
   }
@@ -149,6 +183,13 @@ class LlmLlmObsPlugin extends LLMObsPlugin {
     return { kind: 'llm', name: ctx.model, modelName, modelProvider, sessionId: ctx.sessionId }
   }
 
+  /**
+   * @override
+   */
+  getGenAiApmEndTags (ctx) {
+    return { metrics: extractUsageMetrics(ctx.usage) }
+  }
+
   end (ctx) {
     super.end(ctx)
     super.asyncEnd(ctx)
@@ -158,31 +199,32 @@ class LlmLlmObsPlugin extends LLMObsPlugin {
     const span = ctx.currentStore?.span
     if (!span) return
 
-    const { chunks, llmStartIdx, llmEndIdx, parentToolUseId, initialPrompt, usage } = ctx
+    const { chunks, llmStartIdx, llmEndIdx, parentToolUseId, initialPrompt, systemPrompt, usage } = ctx
 
     if (chunks) {
-      const inputMessages = this.#buildInputMessages(chunks, llmStartIdx, parentToolUseId, initialPrompt)
+      const inputMessages = this.#buildInputMessages(chunks, llmStartIdx, parentToolUseId, initialPrompt, systemPrompt)
       const outputMessages = buildOutputMessages(chunks, llmStartIdx, llmEndIdx)
       this._tagger.tagLLMIO(span, inputMessages, outputMessages)
     }
 
-    if (usage) {
-      const cacheWriteTokens = usage.cache_creation_input_tokens ?? 0
-      const cacheReadTokens = usage.cache_read_input_tokens ?? 0
-      const inputTokens = (usage.input_tokens ?? 0) + cacheWriteTokens + cacheReadTokens
-      const outputTokens = usage.output_tokens ?? 0
-      this._tagger.tagMetrics(span, {
-        input_tokens: inputTokens,
-        output_tokens: outputTokens,
-        cache_read_input_tokens: cacheReadTokens,
-        cache_write_input_tokens: cacheWriteTokens,
-        total_tokens: inputTokens + outputTokens,
-      })
-    }
+    const metrics = extractUsageMetrics(usage)
+    if (metrics) this._tagger.tagMetrics(span, metrics)
   }
 
-  #buildInputMessages (chunks, llmStartIdx, parentToolUseId, initialPrompt) {
+  #buildInputMessages (chunks, llmStartIdx, parentToolUseId, initialPrompt, systemPrompt) {
     const messages = []
+    let configuredPrompt = systemPrompt
+    if (systemPrompt?.type === 'custom') configuredPrompt = systemPrompt.prompt
+    else if (systemPrompt?.type === 'preset') configuredPrompt = systemPrompt.append
+    if (typeof configuredPrompt === 'string') {
+      if (configuredPrompt) messages.push({ role: 'system', content: configuredPrompt })
+    } else if (Array.isArray(configuredPrompt)) {
+      for (const part of configuredPrompt) {
+        if (typeof part === 'string' && part && part !== SYSTEM_PROMPT_DYNAMIC_BOUNDARY) {
+          messages.push({ role: 'system', content: part })
+        }
+      }
+    }
     if (initialPrompt) messages.push({ role: 'user', content: initialPrompt })
     const seenIds = new Set()
 
@@ -245,6 +287,13 @@ class ToolLlmObsPlugin extends LLMObsPlugin {
     super.asyncEnd(ctx)
   }
 
+  /**
+   * @override
+   */
+  getGenAiApmEndTags (ctx) {
+    return subagentToolIds.delete(ctx.id) ? { spanKind: 'agent' } : {}
+  }
+
   setLLMObsTags (ctx) {
     const span = ctx.currentStore?.span
     if (!span) return

package/packages/dd-trace/src/llmobs/plugins/genai/index.js

@@ -23,6 +23,13 @@ class GenAiLLMObsPlugin extends LLMObsPlugin {
 
     // Subscribe to streaming chunk events
     this.addSub('apm:google:genai:request:chunk', ({ ctx, chunk, done }) => {
+      if (!this._llmobsEnabledFor(ctx)) {
+        // only the token usage is needed, for the `gen_ai.usage.*` metrics. The aggregated
+        // response is left alone: it feeds the LLMObs payload and `google_genai.response.model`.
+        if (chunk?.usageMetadata) ctx.streamedUsageMetadata = chunk.usageMetadata
+        return
+      }
+
       ctx.isStreaming = true
       ctx.chunks ||= []
 
@@ -49,6 +56,13 @@ class GenAiLLMObsPlugin extends LLMObsPlugin {
     }
   }
 
+  /**
+   * @override
+   */
+  getGenAiApmEndTags (ctx) {
+    return { metrics: extractMetrics(ctx.result ?? { usageMetadata: ctx.streamedUsageMetadata }) }
+  }
+
   setLLMObsTags (ctx) {
     const { args, methodName } = ctx
     const span = ctx.currentStore?.span