dd-trace 6.7.0 → 6.9.0

package/packages/dd-trace/src/guardrails/apply-pm2-env.js

@@ -0,0 +1,16 @@
+'use strict'
+
+// In PM2 cluster mode, per-app env vars arrive as a `pm2_env`
+// JSON string after preloads have run, so extract them first.
+var pm2EnvStr = process.env.pm2_env
+if (typeof pm2EnvStr === 'string') {
+  try {
+    var pm2Config = JSON.parse(pm2EnvStr)
+    var pm2Keys = Object.keys(pm2Config)
+    for (var i = 0; i < pm2Keys.length; i++) {
+      var key = pm2Keys[i]
+      var value = pm2Config[key]
+      process.env[key] = String(value)
+    }
+  } catch (error) {}
+}

package/packages/dd-trace/src/guardrails/log.js

@@ -2,21 +2,12 @@
 
 /* eslint-disable no-console */
 
+var logLevels = require('../log/levels')
 var isTrue = require('./util').isTrue
 
 var DD_TRACE_DEBUG = process.env.DD_TRACE_DEBUG
 var DD_TRACE_LOG_LEVEL = process.env.DD_TRACE_LOG_LEVEL
 
-var logLevels = {
-  trace: 20,
-  debug: 20,
-  info: 30,
-  warn: 40,
-  error: 50,
-  critical: 50,
-  off: 100
-}
-
 var logLevel = isTrue(DD_TRACE_DEBUG)
   ? Number(DD_TRACE_LOG_LEVEL) || logLevels.debug
   : logLevels.off

package/packages/dd-trace/src/id.js

@@ -185,7 +185,7 @@ function toNumberString (buffer, radix) {
   let low = readInt32(buffer, buffer.length - 4)
   let str = ''
 
-  radix = radix || 10
+  radix ||= 10
 
   while (1) {
     const mod = (high % radix) * UINT_MAX + low

package/packages/dd-trace/src/llmobs/constants/tags.js

@@ -1,7 +1,7 @@
 'use strict'
 
 module.exports = {
-  SPAN_KINDS: ['llm', 'agent', 'workflow', 'task', 'tool', 'embedding', 'retrieval'],
+  SPAN_KINDS: ['llm', 'agent', 'workflow', 'task', 'tool', 'embedding', 'retrieval', 'experiment'],
   SPAN_KIND: '_ml_obs.meta.span.kind',
   SESSION_ID: '_ml_obs.session_id',
   SESSION_ID_TRACE_DEFAULT_KEY: '_ml_obs.trace_session_id',

package/packages/dd-trace/src/llmobs/eval-metric.js

@@ -0,0 +1,249 @@
+'use strict'
+
+const tracerVersion = require('../../../../package.json').version
+
+/**
+ * Kind of event submitted through the evaluation metrics intake. Both kinds share the
+ * writer, the endpoint and most of their validation, and are told apart on the wire by
+ * the `event_kind` field.
+ * @typedef {'evaluation' | 'feedback'} MetricEventKind
+ */
+
+/**
+ * Tag value the intake accepts: a string, a nullish value, or anything the SDK can coerce
+ * through `toString()`.
+ * @typedef {string | null | undefined | { toString?: unknown }} MetricTagValue
+ */
+
+/** @type {string[]} */
+const EVALUATION_METRIC_TYPES = ['categorical', 'score', 'boolean', 'json']
+
+/**
+ * End-user feedback additionally accepts free-form text values.
+ * @type {string[]}
+ */
+const FEEDBACK_METRIC_TYPES = [...EVALUATION_METRIC_TYPES, 'text']
+
+/** @type {Record<MetricEventKind, string[]>} */
+const METRIC_TYPES = {
+  evaluation: EVALUATION_METRIC_TYPES,
+  feedback: FEEDBACK_METRIC_TYPES,
+}
+
+/**
+ * Feedback target option accepted by the SDK, mapped to the payload key it is sent as. `span` and
+ * `spanId` are two ways of naming the same target.
+ * @type {Record<string, string>}
+ */
+const FEEDBACK_TARGET_KEYS = {
+  span: 'span_id',
+  spanId: 'span_id',
+  traceId: 'trace_id',
+  sessionId: 'session_id',
+  feedbackJoinKey: 'feedback_join_key',
+}
+
+// The validation is shared but the user-facing wording is per-kind. Keeping the terms in one
+// table avoids threading three near-identical strings through every validator.
+/** @type {Record<MetricEventKind, { metric: string, metrics: string, data: string }>} */
+const TERMS = {
+  evaluation: {
+    metric: 'evaluation metric',
+    metrics: 'evaluation metrics',
+    data: 'Evaluation metric data',
+  },
+  feedback: {
+    metric: 'feedback metric',
+    metrics: 'feedback metrics',
+    data: 'Feedback data',
+  },
+}
+
+/**
+ * @param {string[]} metricTypes
+ * @returns {string} the quoted list, e.g. `"categorical", "score" or "json"`
+ */
+function formatMetricTypes (metricTypes) {
+  const quoted = metricTypes.map(metricType => `"${metricType}"`)
+  return `${quoted.slice(0, -1).join(', ')} or ${quoted.at(-1)}`
+}
+
+// Built once at load time: only ever used to build an error message.
+/** @type {Record<MetricEventKind, string>} */
+const METRIC_TYPES_MESSAGE = {
+  evaluation: formatMetricTypes(EVALUATION_METRIC_TYPES),
+  feedback: formatMetricTypes(FEEDBACK_METRIC_TYPES),
+}
+
+/**
+ * Builds an error carrying the telemetry error tag the SDK reports for the failed submission.
+ * @param {string} errorTag - Telemetry `error_type` tag value.
+ * @param {string} message - User-facing message.
+ * @returns {Error} the tagged error, to be thrown by the caller.
+ */
+function taggedError (errorTag, message) {
+  const error = new Error(message)
+  // Same shape as the tagger's failures, which the SDK already reads back as `e.ddErrorTag`.
+  Object.defineProperty(error, 'ddErrorTag', { get () { return errorTag } })
+  return error
+}
+
+/**
+ * @param {MetricEventKind} kind
+ * @returns {Error} the tagged error, to be thrown by the caller.
+ */
+function invalidTagsError (kind) {
+  return taggedError('invalid_tags', `Failed to parse tags. Tags for ${TERMS[kind].metrics} must be strings`)
+}
+
+/**
+ * @param {unknown} timestampMs
+ * @param {MetricEventKind} kind
+ * @returns {void}
+ */
+function validateTimestamp (timestampMs, kind) {
+  if (typeof timestampMs !== 'number' || timestampMs < 0) {
+    throw taggedError(
+      'invalid_timestamp',
+      `timestampMs must be a non-negative integer. ${TERMS[kind].data} will not be sent`
+    )
+  }
+}
+
+/**
+ * Validates the label and returns it as sent on the wire.
+ * @param {unknown} label
+ * @param {MetricEventKind} kind
+ * @returns {unknown} the label, coerced to a string for feedback and left untouched for evaluations.
+ */
+function validateLabel (label, kind) {
+  if (!label) {
+    throw taggedError('invalid_metric_label', `label must be the specified name of the ${TERMS[kind].metric}`)
+  }
+
+  // Evaluations keep the looser check they shipped with: a dotted or non-string label reaches the
+  // intake as-is. Aligning them with feedback rejects calls that succeed today, so it is deferred.
+  if (kind === 'evaluation') return label
+
+  // A dot makes the label unusable as a facet key.
+  const labelValue = String(label)
+  if (labelValue.includes('.')) {
+    throw taggedError('invalid_label_value', 'label value must not contain a "."')
+  }
+
+  return labelValue
+}
+
+/**
+ * @param {string | undefined} metricType - Already lower-cased metric type.
+ * @param {MetricEventKind} kind
+ * @returns {void}
+ */
+function validateMetricType (metricType, kind) {
+  if (!metricType || !METRIC_TYPES[kind].includes(metricType)) {
+    throw taggedError('invalid_metric_type', `metricType must be one of ${METRIC_TYPES_MESSAGE[kind]}`)
+  }
+}
+
+/**
+ * @param {string | undefined} metricType - Metric type already validated for the submitted kind.
+ * @param {unknown} value
+ * @returns {void}
+ */
+function validateMetricValue (metricType, value) {
+  if (metricType === 'categorical' && typeof value !== 'string') {
+    throw taggedError('invalid_metric_value', 'value must be a string for a categorical metric.')
+  }
+  if (metricType === 'score' && typeof value !== 'number') {
+    throw taggedError('invalid_metric_value', 'value must be a number for a score metric.')
+  }
+  if (metricType === 'boolean' && typeof value !== 'boolean') {
+    throw taggedError('invalid_metric_value', 'value must be a boolean for a boolean metric')
+  }
+  if (metricType === 'json' && (typeof value !== 'object' || value == null || Array.isArray(value))) {
+    throw taggedError('invalid_metric_value', 'value must be a JSON object for a json metric')
+  }
+  if (metricType === 'text' && typeof value !== 'string') {
+    throw taggedError('invalid_metric_value', 'value must be a string for a text metric')
+  }
+}
+
+/**
+ * @param {unknown} assessment
+ * @returns {void}
+ */
+function validateAssessment (assessment) {
+  if (assessment != null && assessment !== 'pass' && assessment !== 'fail') {
+    throw taggedError('invalid_assessment', 'assessment must be pass or fail')
+  }
+}
+
+/**
+ * @param {unknown} reasoning
+ * @returns {void}
+ */
+function validateReasoning (reasoning) {
+  if (reasoning != null && typeof reasoning !== 'string') {
+    throw taggedError('invalid_reasoning', 'reasoning must be a string')
+  }
+}
+
+/**
+ * Serializes the user-provided tags into the `key:value` list expected by the intake, along
+ * with the tags every submission carries.
+ * @param {Record<string, MetricTagValue> | undefined} tags - User-provided tags, coerced when not strings.
+ * @param {string} mlApp - Resolved ML app name.
+ * @param {MetricEventKind} kind
+ * @param {boolean} [otelEnabled] - Adds `source:otel` so the backend waits for OTel span conversion.
+ * @returns {string[]} the serialized tag list.
+ */
+function buildMetricTags (tags, mlApp, kind, otelEnabled = false) {
+  // An array or a string would otherwise be walked by index and produce tags named after
+  // their offsets, which the intake would happily store.
+  if (tags != null && (typeof tags !== 'object' || Array.isArray(tags))) {
+    throw invalidTagsError(kind)
+  }
+
+  const metricTags = {
+    'ddtrace.version': tracerVersion,
+    ml_app: mlApp,
+  }
+
+  if (tags) {
+    for (const key of Object.keys(tags)) {
+      const tag = tags[key]
+      if (typeof tag === 'string') {
+        metricTags[key] = tag
+      } else if (tag == null) {
+        // A nullish value can be intentional, so it is serialized rather than rejected. Checked
+        // before the `toString` lookup below, which would throw a raw TypeError on it.
+        metricTags[key] = String(tag)
+      } else if (typeof tag.toString === 'function') {
+        metricTags[key] = tag.toString()
+      } else {
+        // Every other value carries a `toString`, either its own or the one inherited from
+        // `Object.prototype`, so this is only reached for a null-prototype object.
+        throw invalidTagsError(kind)
+      }
+    }
+  }
+
+  if (otelEnabled) {
+    metricTags.source = 'otel'
+  }
+
+  return Object.entries(metricTags).map(([key, value]) => `${key}:${value}`)
+}
+
+module.exports = {
+  EVALUATION_METRIC_TYPES,
+  FEEDBACK_METRIC_TYPES,
+  FEEDBACK_TARGET_KEYS,
+  buildMetricTags,
+  validateAssessment,
+  validateLabel,
+  validateMetricType,
+  validateMetricValue,
+  validateReasoning,
+  validateTimestamp,
+}

package/packages/dd-trace/src/llmobs/experiments/client.js

@@ -14,9 +14,11 @@ function apiHost (site) {
 }
 
 // Web-app host for dashboard URLs. Single-level sites (datadoghq.com,
-// ddog-gov.com) are served from the `app.` subdomain; regional sites
-// (us3.datadoghq.com, ap1.datadoghq.com) are used as-is.
+// ddog-gov.com) are served from the `app.` subdomain; staging uses
+// dd.datad0g.com; regional sites (us3.datadoghq.com, ap1.datadoghq.com)
+// are used as-is.
 function appHost (site) {
+  if (site === 'datad0g.com') return 'dd.datad0g.com'
   return site.split('.').length === 2 ? `app.${site}` : site
 }
 

package/packages/dd-trace/src/llmobs/experiments/dataset.js

@@ -2,15 +2,35 @@
 
 const { API_BASE_PATH } = require('./client')
 
-// Immutable dataset record: { input, expectedOutput?, metadata? }.
+// Dataset record: { input, expectedOutput?, metadata?, id? }.
+// `id` may be user-provided before push or filled from the backend-created record.
 class DatasetRecord {
-  constructor (input, expectedOutput = null, metadata = {}) {
+  constructor (input, expectedOutput = null, metadata = {}, id = null) {
     this.input = input
     this.expectedOutput = expectedOutput ?? null
     this.metadata = metadata ?? {}
+    this.id = id ?? null
   }
 }
 
+function createdRecordsFromResponse (response) {
+  if (Array.isArray(response?.records)) return response.records
+  if (Array.isArray(response?.data)) return response.data
+  return []
+}
+
+function recordIdFromCreatedRecord (record) {
+  return String(record?.id ?? record?.attributes?.id ?? '')
+}
+
+function versionFromCreatedRecords (records) {
+  const versions = records
+    .map(record => Number(record?.attributes?.valid_from_version ?? record?.attributes?.version))
+    .filter(Number.isFinite)
+  if (versions.length === 0) return null
+  return Math.max(...versions)
+}
+
 // A local buffer of dataset records, created remotely and pushed on first run
 // (or eagerly via push()). Pushes are incremental.
 class Dataset {
@@ -22,6 +42,8 @@ class Dataset {
   #id
   #projectId
   #pushedCount
+  #version
+  #latestVersion
 
   constructor (client, name, description = '') {
     this.#client = client
@@ -32,16 +54,20 @@ class Dataset {
     this.#id = null
     this.#projectId = null
     this.#pushedCount = 0
+    this.#version = null
+    this.#latestVersion = null
   }
 
   // Build a Dataset that already exists remotely (used by pullDataset).
-  static fromExisting (client, name, description, id, projectId, records, recordIds) {
+  static fromExisting (client, name, description, id, projectId, records, recordIds, version, latestVersion) {
     const dataset = new Dataset(client, name, description)
     dataset.#id = id
     dataset.#projectId = projectId
     dataset.#records.push(...records)
     dataset.#recordIds.push(...recordIds)
     dataset.#pushedCount = records.length
+    dataset.#version = version ?? null
+    dataset.#latestVersion = latestVersion ?? version ?? null
     return dataset
   }
 
@@ -74,6 +100,14 @@ class Dataset {
     return this.#projectId
   }
 
+  version () {
+    return this.#version
+  }
+
+  latestVersion () {
+    return this.#latestVersion
+  }
+
   // Dashboard URL for this dataset, or null until pushed/pulled.
   url () {
     if (this.#id === null) return null
@@ -100,7 +134,12 @@ class Dataset {
         throw new Error(`Failed to create dataset '${this.#name}': ${err.message}`)
       }
       this.#id = response?.data?.id ?? null
+      if (this.#id === null) {
+        throw new Error(`Failed to create dataset '${this.#name}': backend response is missing dataset id`)
+      }
       this.#projectId = projectId
+      this.#version = response?.data?.attributes?.current_version ?? this.#version
+      this.#latestVersion = response?.data?.attributes?.current_version ?? this.#latestVersion
     }
 
     if (this.#pushedCount >= this.#records.length) return { pushedCount: 0, totalCount: 0 }
@@ -108,6 +147,9 @@ class Dataset {
     const pending = this.#records.slice(this.#pushedCount)
     const records = pending.map((rec) => {
       const out = { input: rec.input }
+      if (rec.id != null) {
+        out.id = rec.id
+      }
       if (rec.expectedOutput !== null && rec.expectedOutput !== undefined) {
         out.expected_output = rec.expectedOutput
       }
@@ -128,20 +170,30 @@ class Dataset {
       throw new Error(`Failed to push records to dataset '${this.#name}': ${err.message}`)
     }
 
-    // The append-records response returns created records under a top-level
-    // `records` field, not the usual `data` envelope.
-    const created = response?.records
+    // The append-records response has used both a top-level `records` array
+    // and JSON:API `data` resources. Accept either so generated/custom record
+    // ids are preserved for experiment row tagging.
+    const created = createdRecordsFromResponse(response)
+    const pushedVersion = versionFromCreatedRecords(created)
+    if (pushedVersion === null) {
+      // The dataset contents changed, but the backend did not report the new
+      // version. Avoid pinning later experiments to the pre-append create version.
+      this.#version = null
+    } else {
+      this.#version = pushedVersion
+      this.#latestVersion = Math.max(Number(this.#latestVersion ?? pushedVersion), pushedVersion)
+    }
+
     let pushedCount = 0
-    if (Array.isArray(created)) {
-      for (const node of created) {
-        const recordId = String(node?.id ?? '')
-        if (recordId !== '') pushedCount++
-        this.#recordIds.push(recordId)
+    for (const [index, node] of created.entries()) {
+      const recordId = recordIdFromCreatedRecord(node)
+      if (recordId !== '') {
+        pushedCount++
+        pending[index].id = recordId
       }
-      for (let i = created.length; i < pending.length; i++) this.#recordIds.push('')
-    } else {
-      for (let i = 0; i < pending.length; i++) this.#recordIds.push('')
+      this.#recordIds.push(recordId)
     }
+    for (let i = created.length; i < pending.length; i++) this.#recordIds.push('')
 
     // Advance by the snapshotted pending count, not the live records length,
     // so records added while this push was in flight aren't skipped by the next push.