dd-trace 6.16.0 → 6.18.0

package/packages/dd-trace/src/llmobs/prompts/prompt.js

@@ -0,0 +1,158 @@
+'use strict'
+
+/** @typedef {import('../../../../../index').llmobs.FormattedPromptMessage} FormattedPromptMessage */
+
+const VARIABLE_PATTERN = /(?<!\{)(?:\{\{\s*(\w+)\s*\}\}(?!\})|\{\s*(\w+)\s*\}(?!\}))/g
+
+function isMessage (value) {
+  return typeof value?.role === 'string' &&
+    value?.type !== 'placeholder' &&
+    (value?.content == null || typeof value.content === 'string') &&
+    (typeof value?.content === 'string' ||
+      (Array.isArray(value?.tool_calls) && value.tool_calls.length > 0) ||
+      (Array.isArray(value?.tool_results) && value.tool_results.length > 0))
+}
+
+function isPlaceholder (value) {
+  return value?.type === 'placeholder' && typeof value?.name === 'string'
+}
+
+function isObject (value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value)
+}
+
+function isTool (value) {
+  return isObject(value) &&
+    (value.id == null || typeof value.id === 'string') &&
+    (value.type == null || typeof value.type === 'string') &&
+    (value.tool_id == null || typeof value.tool_id === 'string') &&
+    (value.name == null || typeof value.name === 'string')
+}
+
+function hasValidTools (message) {
+  return (message.tool_call_id == null || typeof message.tool_call_id === 'string') &&
+    (message.tool_calls == null || (Array.isArray(message.tool_calls) && message.tool_calls.every(call =>
+      isTool(call) && (call.function == null ||
+        (isObject(call.function) &&
+          typeof call.function.name === 'string' && typeof call.function.arguments === 'string'))))) &&
+    (message.tool_results == null || (Array.isArray(message.tool_results) && message.tool_results.every(isTool)))
+}
+
+function render (template, variables) {
+  return template.replaceAll(VARIABLE_PATTERN, (match, doubleName, singleName) => {
+    const name = doubleName ?? singleName
+    return Object.hasOwn(variables, name) ? String(variables[name]) : match
+  })
+}
+
+function freezeConfig (config) {
+  if (config === undefined) return Object.freeze({})
+  if (!config || typeof config !== 'object' || Array.isArray(config)) {
+    throw new TypeError('Invalid prompt config: expected a JSON object')
+  }
+  const copy = structuredClone(config)
+  const freeze = value => {
+    if (!value || typeof value !== 'object' || Object.isFrozen(value)) return value
+    for (const nested of Object.values(value)) freeze(nested)
+    return Object.freeze(value)
+  }
+  return freeze(copy)
+}
+class ManagedPrompt {
+  /**
+   * @param {object} data
+   * @param {string} data.id
+   * @param {string} data.version
+   * @param {'registry'|'cache'|'fallback'|'ff'|'resolve'} data.source
+   * @param {string | Array<{role: string, content: string} | {type: 'placeholder', name: string}>} data.template
+   * @param {Record<string, unknown>} [data.config]
+   * @param {string} [data.promptUuid]
+   * @param {string} [data.promptVersionUuid]
+   */
+  constructor ({ id, version, source, template, config, promptUuid, promptVersionUuid }) {
+    this.id = id
+    this.version = version
+    this.source = source
+    this.template = Array.isArray(template)
+      ? Object.freeze(template.map(item => Object.freeze(isPlaceholder(item)
+        ? { type: 'placeholder', name: item.name }
+        : { role: item.role, content: item.content })))
+      : template
+    this.config = freezeConfig(config)
+    this.promptUuid = promptUuid
+    this.promptVersionUuid = promptVersionUuid
+    Object.freeze(this)
+  }
+
+  /**
+   * Render the prompt without changing its stored template.
+   * @param {Record<string, unknown>} [variables]
+   * @returns {string | FormattedPromptMessage[]}
+   */
+  format (variables = {}) {
+    if (typeof this.template === 'string') return render(this.template, variables)
+    return this.template.flatMap(item => {
+      if (!isPlaceholder(item)) return [{ ...item, content: render(item.content, variables) }]
+      if (!Object.hasOwn(variables, item.name)) {
+        throw new TypeError(`Missing message placeholder variable '${item.name}'`)
+      }
+      const messages = variables[item.name]
+      if (!Array.isArray(messages) || !messages.every(message => isMessage(message) && hasValidTools(message))) {
+        throw new TypeError(`Invalid message placeholder variable '${item.name}': expected an array of messages`)
+      }
+      return structuredClone(messages)
+    })
+  }
+
+  /**
+   * Convert the managed prompt to the existing public annotation shape.
+   * @param {Record<string, unknown>} [variables]
+   * @returns {Record<string, unknown>}
+   */
+  toAnnotation (variables) {
+    const annotation = {
+      id: this.id,
+      version: this.version,
+      template: typeof this.template === 'string' ? this.template : this.template.map(message => ({ ...message })),
+    }
+    const placeholderNames = new Set(Array.isArray(this.template)
+      ? this.template.filter(isPlaceholder).map(item => item.name)
+      : [])
+    const entries = Object.entries(variables ?? {}).filter(([name]) => !placeholderNames.has(name))
+    if (entries.length) {
+      annotation.variables = Object.fromEntries(entries.map(([name, value]) => [name, String(value)]))
+    }
+    if (this.promptUuid) annotation.promptUuid = this.promptUuid
+    if (this.promptVersionUuid) annotation.promptVersionUuid = this.promptVersionUuid
+    return annotation
+  }
+
+  /**
+   * Convert a caller fallback to a managed prompt.
+   * @param {string} promptId
+   * @param {string | object | Array<{role: string, content: string} | {type: 'placeholder', name: string}> |
+   *   (() => string | object | Array<{role: string, content: string} | {type: 'placeholder', name: string}>)} fallback
+   * @returns {ManagedPrompt}
+   */
+  static fromFallback (promptId, fallback) {
+    const value = typeof fallback === 'function' ? fallback() : fallback
+    const promptLike = value && !Array.isArray(value) && typeof value === 'object'
+    const template = promptLike ? value.template : value
+    const validTemplate = typeof template === 'string' || (
+      Array.isArray(template) && template.every(item =>
+        isPlaceholder(item) || (isMessage(item) && typeof item.content === 'string'))
+    )
+    if (!validTemplate) {
+      throw new TypeError('Invalid prompt fallback: expected a string, chat message array, or object with a template')
+    }
+    return new ManagedPrompt({
+      id: promptId,
+      version: String(promptLike && value.version ? value.version : 'fallback'),
+      source: 'fallback',
+      template,
+      config: promptLike ? value.config : undefined,
+    })
+  }
+}
+
+module.exports = ManagedPrompt

package/packages/dd-trace/src/llmobs/sdk.js

@@ -31,6 +31,7 @@ const { storage } = require('./storage')
 const telemetry = require('./telemetry')
 const LLMObsTagger = require('./tagger')
 const { createExperiments } = require('./experiments')
+const PromptManager = require('./prompts/manager')
 
 // communicating with writer
 const evalMetricAppendCh = channel('llmobs:eval-metric:append')
@@ -45,12 +46,17 @@ class LLMObs extends NoopLLMObs {
    */
   #hasUserSpanProcessor = false
 
+  #promptManager
+
+  #getOpenFeatureProvider
+
   /**
    * @param {import('../tracer')} tracer - Tracer instance
    * @param {import('./index')} llmobsModule - LLMObs module instance
    * @param {import('../config/config-base')} config - Tracer configuration
+   * @param {() => object} getOpenFeatureProvider - Lazy getter for the tracer's existing OpenFeature provider
    */
-  constructor (tracer, llmobsModule, config) {
+  constructor (tracer, llmobsModule, config, getOpenFeatureProvider = () => {}) {
     super(tracer)
 
     /** @type {import('../config/config-base')} */
@@ -58,6 +64,7 @@ class LLMObs extends NoopLLMObs {
 
     this._llmobsModule = llmobsModule
     this._tagger = new LLMObsTagger(config)
+    this.#getOpenFeatureProvider = getOpenFeatureProvider
   }
 
   get enabled () {
@@ -73,6 +80,15 @@ class LLMObs extends NoopLLMObs {
     return createExperiments(this._config, this)
   }
 
+  /**
+   * Prompt Management API.
+   * @returns {import('../../../../index').llmobs.Prompts}
+   */
+  get prompts () {
+    this.#promptManager ??= new PromptManager(this._config, this.#getOpenFeatureProvider)
+    return this.#promptManager
+  }
+
   enable (options = {}) {
     logger.warn(
       'Enabling LLM Observability via `llmobs.enable()` is deprecated and will be removed in dd-trace@7.0.0. ' +
@@ -97,8 +113,8 @@ class LLMObs extends NoopLLMObs {
 
     // TODO: These configs should be passed through directly at construction time instead.
     this._config.llmobs.DD_LLMOBS_ENABLED = true
-    this._config.llmobs.mlApp = options.mlApp
-    this._config.llmobs.agentlessEnabled = options.agentlessEnabled
+    this._config.llmobs.DD_LLMOBS_ML_APP = options.mlApp
+    this._config.llmobs.DD_LLMOBS_AGENTLESS_ENABLED = options.agentlessEnabled
 
     // configure writers and channel subscribers
     this._llmobsModule.enable(this._config)
@@ -393,7 +409,7 @@ class LLMObs extends NoopLLMObs {
           'spanId and traceId must both be specified for the given evaluation metric to be submitted.'
         )
       }
-      const mlApp = options.mlApp || this._config.llmobs.mlApp
+      const mlApp = options.mlApp || this._config.llmobs.DD_LLMOBS_ML_APP
       if (!mlApp) {
         err = 'missing_ml_app'
         throw new Error(
@@ -475,7 +491,6 @@ class LLMObs extends NoopLLMObs {
    * @param {number} [options.timestampMs] - When the feedback was generated. Defaults to now.
    * @param {'pass' | 'fail'} [options.assessment] - Assessment of the feedback.
    * @param {string} [options.reasoning] - Explanation of the feedback.
-   * @returns {void}
    */
   submitFeedback (options = {}) {
     if (!this.enabled) return
@@ -532,7 +547,7 @@ class LLMObs extends NoopLLMObs {
         throw new TypeError('submitter.type must be a string')
       }
 
-      const mlApp = options.mlApp || this._config.llmobs.mlApp
+      const mlApp = options.mlApp || this._config.llmobs.DD_LLMOBS_ML_APP
       if (!mlApp) {
         err = 'missing_ml_app'
         throw new Error('ML App name is required for sending feedback. Feedback data will not be sent.')

package/packages/dd-trace/src/llmobs/span_processor.js

@@ -38,11 +38,41 @@ const {
   SAMPLE_RATE,
   SAMPLING_DECISION,
   TRACE_ID,
+  INPUT_TOKENS_METRIC_KEY,
+  OUTPUT_TOKENS_METRIC_KEY,
+  TOTAL_TOKENS_METRIC_KEY,
+  CACHE_READ_INPUT_TOKENS_METRIC_KEY,
+  CACHE_WRITE_INPUT_TOKENS_METRIC_KEY,
+  REASONING_OUTPUT_TOKENS_METRIC_KEY,
+  GEN_AI_OPERATION_NAME,
+  GEN_AI_REQUEST_MODEL,
+  GEN_AI_PROVIDER_NAME,
+  GEN_AI_APPLICATION_NAME,
+  GEN_AI_CONVERSATION_ID,
+  GEN_AI_USAGE_INPUT_TOKENS_METRIC_KEY,
+  GEN_AI_USAGE_OUTPUT_TOKENS_METRIC_KEY,
+  GEN_AI_USAGE_TOTAL_TOKENS_METRIC_KEY,
+  GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_METRIC_KEY,
+  GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_METRIC_KEY,
+  GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_METRIC_KEY,
+  ARTIFICIAL_GEN_AI_TAGS,
 } = require('./constants/tags')
 const { UNSERIALIZABLE_VALUE_TEXT } = require('./constants/text')
 const telemetry = require('./telemetry')
 const LLMObsTagger = require('./tagger')
 
+const DEFAULT_MODEL = 'custom'
+const MODEL_BACKED_SPAN_KINDS = new Set(['llm', 'embedding'])
+
+const GEN_AI_TOKEN_METRIC_KEYS = [
+  [INPUT_TOKENS_METRIC_KEY, GEN_AI_USAGE_INPUT_TOKENS_METRIC_KEY],
+  [OUTPUT_TOKENS_METRIC_KEY, GEN_AI_USAGE_OUTPUT_TOKENS_METRIC_KEY],
+  [TOTAL_TOKENS_METRIC_KEY, GEN_AI_USAGE_TOTAL_TOKENS_METRIC_KEY],
+  [CACHE_READ_INPUT_TOKENS_METRIC_KEY, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_METRIC_KEY],
+  [CACHE_WRITE_INPUT_TOKENS_METRIC_KEY, GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_METRIC_KEY],
+  [REASONING_OUTPUT_TOKENS_METRIC_KEY, GEN_AI_USAGE_REASONING_OUTPUT_TOKENS_METRIC_KEY],
+]
+
 class LLMObservabilitySpan {
   /**
    * @param {string} kind span kind
@@ -90,6 +120,12 @@ class LLMObsSpanProcessor {
     // if the span is not in our private tagger map, it is not an llmobs span
     if (!LLMObsTagger.tagMap.has(span)) return
 
+    try {
+      this.#setGenAiApmTags(span)
+    } catch (e) {
+      logger.debug('Failed to set gen_ai APM tags:', e.message)
+    }
+
     try {
       const formattedEvent = this.format(span)
       telemetry.incrementLLMObsSpanFinishedCount(span)
@@ -138,8 +174,8 @@ class LLMObsSpanProcessor {
     const output = {}
 
     if (['llm', 'embedding'].includes(spanKind)) {
-      meta.model_name = mlObsTags[MODEL_NAME] || 'custom'
-      meta.model_provider = (mlObsTags[MODEL_PROVIDER] || 'custom').toLowerCase()
+      meta.model_name = mlObsTags[MODEL_NAME] || DEFAULT_MODEL
+      meta.model_provider = (mlObsTags[MODEL_PROVIDER] || DEFAULT_MODEL).toLowerCase()
     }
 
     if (mlObsTags[METADATA] || mlObsTags[COST_TAGS]) {
@@ -283,6 +319,49 @@ class LLMObsSpanProcessor {
     return llmObsSpanEvent
   }
 
+  /**
+   * Writes the scalar `gen_ai.*` attributes onto the APM span, so model, provider, application,
+   * conversation and token usage are searchable in APM. Message bodies stay off the APM span.
+   *
+   * @param {import('../opentracing/span')} span
+   */
+  #setGenAiApmTags (span) {
+    const mlObsTags = LLMObsTagger.tagMap.get(span)
+    const spanContext = span.context()
+    const spanKind = mlObsTags[SPAN_KIND]
+
+    if (spanKind) spanContext.setTag(GEN_AI_OPERATION_NAME, spanKind)
+
+    const modelName = mlObsTags[MODEL_NAME]
+    const modelProvider = mlObsTags[MODEL_PROVIDER]
+    const modelBacked = MODEL_BACKED_SPAN_KINDS.has(spanKind)
+    if (modelBacked) {
+      spanContext.setTag(GEN_AI_REQUEST_MODEL, modelName || DEFAULT_MODEL)
+      spanContext.setTag(GEN_AI_PROVIDER_NAME, (modelProvider || DEFAULT_MODEL).toLowerCase())
+    } else {
+      if (modelName) spanContext.setTag(GEN_AI_REQUEST_MODEL, modelName)
+      if (modelProvider) spanContext.setTag(GEN_AI_PROVIDER_NAME, modelProvider.toLowerCase())
+    }
+
+    const mlApp = mlObsTags[ML_APP]
+    if (mlApp) spanContext.setTag(GEN_AI_APPLICATION_NAME, mlApp)
+
+    const sessionId = mlObsTags[SESSION_ID]
+    if (sessionId) spanContext.setTag(GEN_AI_CONVERSATION_ID, sessionId)
+
+    const metrics = mlObsTags[METRICS]
+    // Other kinds carry unrelated metrics that would be misleading under a `gen_ai.usage.*` key.
+    if (modelBacked && metrics) {
+      for (const [metricKey, genAiKey] of GEN_AI_TOKEN_METRIC_KEYS) {
+        const value = metrics[metricKey]
+        if (value != null) spanContext.setTag(genAiKey, value)
+      }
+    }
+
+    // matches the value dd-trace-py writes
+    spanContext.setTag(ARTIFICIAL_GEN_AI_TAGS, 'true')
+  }
+
   // For now, this only applies to metadata, as we let users annotate this field with any object
   // However, we want to protect against circular references or BigInts (unserializable)
   // This function can be reused for other fields if needed

package/packages/dd-trace/src/llmobs/tagger.js

@@ -87,7 +87,7 @@ class LLMObsTagger {
   }
 
   /**
-   * The sampler reads its rate from `config.llmobs.sampleRate`, which can change
+   * The sampler reads its rate from `config.llmobs.DD_LLMOBS_SAMPLE_RATE`, which can change
    * at runtime (e.g. via remote config). Rebuild the sampler whenever the rate
    * changes so decisions reflect the current config, while reusing the existing
    * sampler when it hasn't.
@@ -95,7 +95,7 @@ class LLMObsTagger {
    * @returns {import('../sampler')}
    */
   #getSampler () {
-    const rate = this.#config.llmobs?.sampleRate ?? 1
+    const rate = this.#config.llmobs?.DD_LLMOBS_SAMPLE_RATE ?? 1
     if (this.#sampler === null || rate !== this.#sampler.rate()) {
       this.#sampler = new Sampler(rate)
     }
@@ -128,7 +128,7 @@ class LLMObsTagger {
       mlApp ||
       registry.get(parent)?.[ML_APP] ||
       span.context()._trace.tags[PROPAGATED_ML_APP_KEY] ||
-      this.#config.llmobs.mlApp ||
+      this.#config.llmobs.DD_LLMOBS_ML_APP ||
       this.#config.service // this should always have a default
 
     if (!spanMlApp) {
@@ -415,6 +415,8 @@ class LLMObsTagger {
       template,
       contextVariables,
       queryVariables,
+      promptUuid,
+      promptVersionUuid,
     } = prompt
 
     if (strictValidation) {
@@ -429,6 +431,8 @@ class LLMObsTagger {
       }
     }
 
+    const currentPrompt = registry.get(span)?.[INPUT_PROMPT]
+    const replacesPrompt = id != null || version != null || template != null
     const finalPromptId = id ?? `${mlApp}_${DEFAULT_PROMPT_NAME}`
     const finalCtxVariablesKeys = contextVariables ?? ['context']
     const finalQueryVariablesKeys = queryVariables ?? ['question']
@@ -471,6 +475,16 @@ class LLMObsTagger {
       return
     }
 
+    if (promptUuid != null && typeof promptUuid !== 'string') {
+      this.#handleFailure('Prompt UUID must be a string.', 'invalid_prompt')
+      return
+    }
+
+    if (promptVersionUuid != null && typeof promptVersionUuid !== 'string') {
+      this.#handleFailure('Prompt version UUID must be a string.', 'invalid_prompt')
+      return
+    }
+
     // validate prompt tags
     if (tags && (typeof tags !== 'object' || tags instanceof Map)) {
       this.#handleFailure('Prompt tags must be an non-Map object.', 'invalid_prompt')
@@ -491,10 +505,13 @@ class LLMObsTagger {
     }
 
     if (Array.isArray(template)) {
-      for (const message of template) {
-        if (typeof message !== 'object' || !message.role || !message.content) {
+      for (const item of template) {
+        const valid = item?.type === 'placeholder'
+          ? typeof item.name === 'string'
+          : typeof item?.role === 'string' && typeof item?.content === 'string'
+        if (!valid) {
           this.#handleFailure(
-            'Prompt chat template must be an array of objects with role and content properties.', 'invalid_prompt'
+            'Prompt chat template must contain messages or message placeholders.', 'invalid_prompt'
           )
           return
         }
@@ -518,21 +535,32 @@ class LLMObsTagger {
     if (typeof template === 'string') {
       finalTemplate = template
     } else if (Array.isArray(template)) {
-      finalChatTemplate = template.map(message => ({ role: message.role, content: message.content }))
+      finalChatTemplate = template.map(item => item.type === 'placeholder'
+        ? { type: 'placeholder', name: item.name }
+        : { role: item.role, content: item.content })
     }
 
     const validatedPrompt = {}
-    if (finalPromptId) validatedPrompt.id = finalPromptId
+    if (finalPromptId && (!currentPrompt || replacesPrompt)) validatedPrompt.id = finalPromptId
     if (version) validatedPrompt.version = version
+    if (promptUuid) validatedPrompt.prompt_uuid = promptUuid
+    if (promptVersionUuid) validatedPrompt.prompt_version_uuid = promptVersionUuid
     if (variables) validatedPrompt.variables = variables
     if (finalTemplate) validatedPrompt.template = finalTemplate
     if (finalChatTemplate?.length) validatedPrompt.chat_template = finalChatTemplate
     if (tags) validatedPrompt.tags = tags
-    if (finalCtxVariablesKeys) validatedPrompt[INTERNAL_CONTEXT_VARIABLE_KEYS] = finalCtxVariablesKeys
-    if (finalQueryVariablesKeys) validatedPrompt[INTERNAL_QUERY_VARIABLE_KEYS] = finalQueryVariablesKeys
+    if (finalCtxVariablesKeys && (!currentPrompt || replacesPrompt || contextVariables != null)) {
+      validatedPrompt[INTERNAL_CONTEXT_VARIABLE_KEYS] = finalCtxVariablesKeys
+    }
+    if (finalQueryVariablesKeys && (!currentPrompt || replacesPrompt || queryVariables != null)) {
+      validatedPrompt[INTERNAL_QUERY_VARIABLE_KEYS] = finalQueryVariablesKeys
+    }
 
-    const currentPrompt = registry.get(span)?.[INPUT_PROMPT]
     if (currentPrompt) {
+      if (replacesPrompt) {
+        if (promptUuid == null) currentPrompt.prompt_uuid = undefined
+        if (promptVersionUuid == null) currentPrompt.prompt_version_uuid = undefined
+      }
       Object.assign(currentPrompt, validatedPrompt)
     } else {
       this._setTag(span, INPUT_PROMPT, validatedPrompt)

package/packages/dd-trace/src/llmobs/telemetry.js

@@ -84,11 +84,11 @@ function recordLLMObsEnabled (startTime, config, value = 1) {
   const autoEnabled = !!config._env?.['llmobs.enabled']
   const tags = {
     error: 0,
-    agentless: Number(config.llmobs.agentlessEnabled),
+    agentless: Number(config.llmobs.DD_LLMOBS_AGENTLESS_ENABLED),
     site: config.site,
     auto: Number(autoEnabled),
-    ml_app: config.llmobs.mlApp,
-    sample_rate: config.llmobs.sampleRate,
+    ml_app: config.llmobs.DD_LLMOBS_ML_APP,
+    sample_rate: config.llmobs.DD_LLMOBS_SAMPLE_RATE,
   }
   llmobsMetrics.count('product_enabled', tags).inc(value)
   llmobsMetrics.distribution('init_time', tags).track(initTimeMs)
@@ -188,7 +188,6 @@ function recordSubmitEvaluation (options, err, value = 1) {
  * @param {string} targetType - The payload key the feedback was attached to, `'other'` if unresolved.
  * @param {string} err - The telemetry error tag, empty when the submission succeeded.
  * @param {number} [value]
- * @returns {void}
  */
 function recordSubmitFeedback (metricType, targetType, err, value = 1) {
   const tags = {
@@ -205,6 +204,36 @@ function recordLLMObsUserProcessorCalled (error, value = 1) {
   llmobsMetrics.count('user_processor_called', tags).inc(value)
 }
 
+/**
+ * @param {'hot_cache'|'warm_cache'|'registry'|'resolve'|'fallback'|'ff'} source
+ * @param {number} [value]
+ */
+function recordPromptSource (source, value = 1) {
+  llmobsMetrics.count('prompt.source', { from: source }).inc(value)
+}
+
+/**
+ * @param {'NotFound'|'FetchError'} errorType
+ * @param {number} [value]
+ */
+function recordPromptFetchError (errorType, value = 1) {
+  llmobsMetrics.count('prompt.fetch.error', { error_type: errorType }).inc(value)
+}
+
+/**
+ * @param {'GET'|'POST'|'PATCH'|'DELETE'} method
+ * @param {string} errorType
+ * @param {number} status
+ * @param {number} [value]
+ */
+function recordPromptCrudError (method, errorType, status, value = 1) {
+  llmobsMetrics.count('prompt.crud.error', {
+    method,
+    error_type: errorType,
+    status: String(status),
+  }).inc(value)
+}
+
 module.exports = {
   recordLLMObsEnabled,
   incrementLLMObsSpanStartCount,
@@ -220,4 +249,7 @@ module.exports = {
   recordSubmitEvaluation,
   recordSubmitFeedback,
   recordLLMObsUserProcessorCalled,
+  recordPromptSource,
+  recordPromptFetchError,
+  recordPromptCrudError,
 }

package/packages/dd-trace/src/llmobs/util.js

@@ -306,7 +306,6 @@ function getFunctionArguments (fn, args = []) {
 // keys), so a name containing `=` is safe and must not be dropped.
 /**
  * @param {string} name
- * @returns {boolean}
  */
 function agentNameWireSafe (name) {
   // Conservative slice of the 512B shared tagset budget, mirroring dd-trace-py.
@@ -413,7 +412,6 @@ function findGenAIAncestorSpanId (span) {
 /**
  * Generate a 128-bit LLMObs trace ID with the span start time encoded in its high bits.
  * @param {number} startTime
- * @returns {string}
  */
 function generateLlmObsTraceId (startTime) {
   const identifier = id()
@@ -455,45 +453,13 @@ function normalizeLlmObsTraceId (traceId) {
   return identifier > MAX_UINT_64 ? identifier.toString(16).padStart(32, '0') : traceId
 }
 
-// Maps an audio `format` (e.g. "wav", "mp3") to a MIME type. Defaults to `audio/wav` when the
-// format is missing. Provider-specific overrides (e.g. OpenAI's mp3 -> audio/mpeg) are passed in
-// via `mimeTypeLookup` so this stays provider-agnostic. A non-string `format` is treated as missing
-// so a malformed auto-instrumented payload can't throw and disable the plugin.
-/**
- * @param {string} fmt
- * @param {Record<string, string>} [mimeTypeLookup]
- * @returns {string}
- */
-function audioMimeTypeFromFormat (fmt, mimeTypeLookup = {}) {
-  fmt = typeof fmt === 'string' ? fmt.trim().toLowerCase() : ''
-  if (!fmt) return 'audio/wav'
-  return mimeTypeLookup[fmt] ?? `audio/${fmt}`
-}
-
-// Builds an audio part from raw audio bytes (base64-encoded) or an existing base64 string. Only
-// Buffer/Uint8Array inputs are base64-encoded; any other shape is passed through so a malformed
-// auto-instrumented payload can't throw (the tagger soft-skips a non-string `content`).
-/**
- * @param {Buffer | Uint8Array | string} data
- * @param {string} mimeType
- * @returns {{ mimeType: string, content: string }}
- */
-function formatAudioPart (data, mimeType) {
-  const content = Buffer.isBuffer(data) || ArrayBuffer.isView(data)
-    ? Buffer.from(data).toString('base64')
-    : data
-  return { mimeType, content }
-}
-
 module.exports = {
   agentNameWireSafe,
-  audioMimeTypeFromFormat,
   encodeUnicode,
   findGenAIAncestorSpanId,
   generateLlmObsTraceId,
   llmObsTraceIdToWire,
   normalizeLlmObsTraceId,
-  formatAudioPart,
   resolveAgentAttribution,
   validateCostTags,
   validateKind,

package/packages/dd-trace/src/llmobs/writers/base.js

@@ -102,7 +102,6 @@ class BaseLLMObsWriter {
    * @param {Record<string, unknown>} event
    * @param {{ apiKey?: string, site?: string }} [routing]
    * @param {number} [byteLength]
-   * @returns {boolean} `true` if the event was buffered, `false` if it was dropped
    * (e.g. the per-routing buffer was full). Callers that depend on the event
    * actually being submitted should check this value.
    */

package/packages/dd-trace/src/llmobs/writers/util.js

@@ -8,10 +8,9 @@ const { fetchAgentInfo } = require('../../agent/info')
 /**
  * @param {import('../../config/config-base')} config
  * @param {(agentless: boolean) => void} setWritersAgentlessValue
- * @returns {void}
  */
 function setAgentStrategy (config, setWritersAgentlessValue) {
-  const agentlessEnabled = config.llmobs.agentlessEnabled
+  const agentlessEnabled = config.llmobs.DD_LLMOBS_AGENTLESS_ENABLED
 
   if (agentlessEnabled != null) {
     setWritersAgentlessValue(agentlessEnabled)

package/packages/dd-trace/src/log/channels.js

@@ -9,6 +9,8 @@ const debugChannel = channel('datadog:log:debug')
 const infoChannel = channel('datadog:log:info')
 const warnChannel = channel('datadog:log:warn')
 const errorChannel = channel('datadog:log:error')
+// Telemetry needs the unformatted message, cause and transmission policy, not console output.
+const errorRecordChannel = channel('datadog:log:error:record')
 
 const defaultLevel = Level.debug
 
@@ -67,4 +69,5 @@ module.exports = {
   infoChannel,
   warnChannel,
   errorChannel,
+  errorRecordChannel,
 }