dd-trace 5.88.0 → 5.89.0

package/packages/dd-trace/src/datastreams/checkpointer.js

@@ -60,6 +60,19 @@ class DataStreamsCheckpointer {
 
     return ctx
   }
+
+  /**
+   * Records a transaction ID at a named checkpoint without pathway propagation.
+   * Tags the active span (or the provided span) with the transaction ID and checkpoint name.
+   * @param {string} transactionId - The transaction identifier to track.
+   * @param {string} checkpointName - The logical checkpoint name.
+   * @param {object|null} [span=null] - Span to tag. Defaults to the currently active span.
+   */
+  trackTransaction (transactionId, checkpointName, span = null) {
+    if (!this.config.dsmEnabled) return
+    const activeSpan = span ?? this.tracer.scope().active()
+    this.dsmProcessor.trackTransaction(transactionId, checkpointName, activeSpan)
+  }
 }
 
 module.exports = {

package/packages/dd-trace/src/datastreams/index.js

@@ -71,6 +71,7 @@ const DsmPathwayCodec = lazyClass(() => require('./pathway').DsmPathwayCodec, []
 const DataStreamsCheckpointer = lazyClass(() => require('./checkpointer').DataStreamsCheckpointer, [
   'setProduceCheckpoint',
   'setConsumeCheckpoint',
+  'trackTransaction',
 ])
 
 /**
@@ -79,6 +80,7 @@ const DataStreamsCheckpointer = lazyClass(() => require('./checkpointer').DataSt
 const DataStreamsManager = lazyClass(() => require('./manager').DataStreamsManager, [
   'setCheckpoint',
   'decodeDataStreamsContext',
+  'trackTransaction',
 ])
 
 // TODO: Are all those methods actually public?
@@ -92,6 +94,7 @@ const DataStreamsProcessor = lazyClass(() => require('./processor').DataStreamsP
   'setCheckpoint',
   'recordOffset',
   'setOffset',
+  'trackTransaction',
   'setUrl',
   'trySampleSchema',
   'canSampleSchema',

package/packages/dd-trace/src/datastreams/manager.js

@@ -22,6 +22,15 @@ class DataStreamsManager {
     DataStreamsContext.setDataStreamsContext(ctx)
     return ctx
   }
+
+  /**
+   * @param {string} transactionId
+   * @param {string} checkpointName
+   * @param {object|null} [span=null]
+   */
+  trackTransaction (transactionId, checkpointName, span = null) {
+    this._dataStreamsProcessor.trackTransaction(transactionId, checkpointName, span)
+  }
 }
 
 module.exports = { DataStreamsManager }

package/packages/dd-trace/src/datastreams/processor.js

@@ -4,7 +4,7 @@ const os = require('os')
 const pkg = require('../../../../package.json')
 
 const { LogCollapsingLowestDenseDDSketch } = require('../../../../vendor/dist/@datadog/sketches-js')
-const { PATHWAY_HASH } = require('../../../../ext/tags')
+const { PATHWAY_HASH, DSM_TRANSACTION_ID, DSM_TRANSACTION_CHECKPOINT } = require('../../../../ext/tags')
 const log = require('../log')
 const processTags = require('../process-tags')
 const propagationHash = require('../propagation-hash')
@@ -69,10 +69,66 @@ class Backlog {
   }
 }
 
+/**
+ * Maps checkpoint name strings to single-byte IDs (1–254).
+ * ID 0 is reserved; 254 is the maximum number of unique names.
+ * Scope is per-processor so IDs are stable across bucket boundaries within a process lifetime.
+ */
+class CheckpointRegistry {
+  /** @type {Map<string, number>} */
+  #nameToId = new Map()
+  #nextId = 1
+  /** @type {Buffer[]} Pre-built [id uint8][nameLen uint8][name bytes] entries, one per registered name. */
+  #entryBuffers = []
+  /** @type {Buffer | null} Cached concat of #entryBuffers; reset when a new name is added. */
+  #encodedKeysCache = null
+
+  /**
+   * Returns the byte ID for the given checkpoint name, assigning one if not seen before.
+   * Returns undefined when registry is full (254 entries exhausted).
+   * @param {string} name
+   * @returns {number | undefined}
+   */
+  getId (name) {
+    const existing = this.#nameToId.get(name)
+    if (existing !== undefined) return existing
+    if (this.#nextId > 254) return
+    const id = this.#nextId++
+    this.#nameToId.set(name, id)
+    // Build the wire entry now with a bounded write so long names never materialise
+    // their full UTF-8 encoding — buf.write() stops at the supplied byte limit.
+    const nameBuf = Buffer.allocUnsafe(255)
+    const nameByteLen = nameBuf.write(name, 0, 255, 'utf8')
+    const entry = Buffer.allocUnsafe(2 + nameByteLen)
+    entry.writeUInt8(id, 0)
+    entry.writeUInt8(nameByteLen, 1)
+    nameBuf.copy(entry, 2, 0, nameByteLen)
+    this.#entryBuffers.push(entry)
+    this.#encodedKeysCache = null
+    return id
+  }
+
+  /**
+   * Returns a Buffer encoding all registered names as [id uint8][nameLen uint8][name bytes].
+   * Names are truncated to 255 UTF-8 bytes.
+   * Result is cached and only recomputed when new names are registered.
+   * @returns {Buffer}
+   */
+  get encodedKeys () {
+    if (this.#encodedKeysCache !== null) return this.#encodedKeysCache
+    this.#encodedKeysCache = this.#entryBuffers.length > 0
+      ? Buffer.concat(this.#entryBuffers)
+      : Buffer.alloc(0)
+    return this.#encodedKeysCache
+  }
+}
+
 class StatsBucket {
   constructor () {
     this._checkpoints = new Map()
     this._backlogs = new Map()
+    /** @type {Buffer[]} Accumulated transaction byte chunks, concatenated lazily. */
+    this._transactionChunks = []
   }
 
   get checkpoints () {
@@ -83,6 +139,16 @@ class StatsBucket {
     return this._backlogs
   }
 
+  /**
+   * Returns the concatenated transaction bytes, or null if no transactions have been added.
+   * Concatenation is deferred to read time to avoid O(N²) copies during accumulation.
+   * @returns {Buffer | null}
+   */
+  get transactions () {
+    if (this._transactionChunks.length === 0) return null
+    return Buffer.concat(this._transactionChunks)
+  }
+
   forCheckpoint ({ hash, parentHash, edgeTags }) {
     let checkpoint = this._checkpoints.get(hash)
     if (!checkpoint) {
@@ -93,6 +159,14 @@ class StatsBucket {
     return checkpoint
   }
 
+  /**
+   * Appends pre-encoded transaction bytes to this bucket.
+   * @param {Buffer} bytes
+   */
+  addTransaction (bytes) {
+    this._transactionChunks.push(bytes)
+  }
+
   /**
    * Conditionally add a backlog to the bucket. If there is currently an offset
    * matching the backlog's tags, overwrite the offset IFF the backlog's offset
@@ -153,6 +227,7 @@ class DataStreamsProcessor {
     this.sequence = 0
     this.flushInterval = flushInterval
     this._schemaSamplers = {}
+    this._checkpointRegistry = new CheckpointRegistry()
 
     if (this.enabled) {
       this.timer = setInterval(this.onInterval.bind(this), flushInterval)
@@ -294,9 +369,47 @@ class DataStreamsProcessor {
     this.recordOffset(backlogData)
   }
 
+  /**
+   * Records a transaction ID at a named checkpoint using the binary wire format shared with Go/Java tracers.
+   *
+   * Wire format per entry: [checkpointId uint8][timestamp int64 big-endian 8 bytes][idLen uint8][id bytes]
+   *
+   * @param {string} transactionId - Truncated to 255 UTF-8 bytes.
+   * @param {string} checkpointName - Mapped to a stable 1-byte ID; silently dropped if registry full.
+   * @param {import('../opentelemetry/span').Span|null} [span=null] - Active span to tag with DSM transaction metadata.
+   */
+  trackTransaction (transactionId, checkpointName, span = null) {
+    if (!this.enabled) {
+      log.warn('trackTransaction called but DD_DATA_STREAMS_ENABLED is not set. Transaction will not be tracked.')
+      return
+    }
+
+    const checkpointId = this._checkpointRegistry.getId(checkpointName)
+    if (checkpointId === undefined) return
+
+    const idBytes = Buffer.from(transactionId, 'utf8').subarray(0, 255)
+    // Multiply as BigInt to avoid precision loss past MAX_SAFE_INTEGER
+    const timestampNs = BigInt(Date.now()) * 1_000_000n
+
+    const entry = Buffer.alloc(1 + 8 + 1 + idBytes.length)
+    entry.writeUInt8(checkpointId, 0)
+    entry.writeBigInt64BE(timestampNs, 1)
+    entry.writeUInt8(idBytes.length, 9)
+    idBytes.copy(entry, 10)
+
+    // Number() cast is safe here: 10s bucket granularity tolerates ~0.5ns precision loss
+    this.bucketFromTimestamp(Number(timestampNs)).addTransaction(entry)
+
+    if (span) {
+      span.setTag(DSM_TRANSACTION_ID, transactionId)
+      span.setTag(DSM_TRANSACTION_CHECKPOINT, checkpointName)
+    }
+  }
+
   _serializeBuckets () {
     // TimeBuckets
     const serializedBuckets = []
+    const registrySnapshot = this._checkpointRegistry.encodedKeys
 
     for (const [timeNs, bucket] of this.buckets.entries()) {
       const points = []
@@ -311,12 +424,21 @@ class DataStreamsProcessor {
       for (const backlog of bucket._backlogs.values()) {
         backlogs.push(backlog.encode())
       }
-      serializedBuckets.push({
+
+      const serializedBucket = {
         Start: BigInt(timeNs),
         Duration: BigInt(this.bucketSizeNs),
         Stats: points,
         Backlogs: backlogs,
-      })
+      }
+
+      const transactions = bucket.transactions
+      if (transactions !== null) {
+        serializedBucket.Transactions = transactions
+        serializedBucket.TransactionCheckpointIds = registrySnapshot
+      }
+
+      serializedBuckets.push(serializedBucket)
     }
 
     this.buckets.clear()
@@ -358,6 +480,7 @@ class DataStreamsProcessor {
 }
 
 module.exports = {
+  CheckpointRegistry,
   DataStreamsProcessor,
   StatsPoint,
   StatsBucket,

package/packages/dd-trace/src/encode/agentless-json.js

@@ -1,14 +1,16 @@
 'use strict'
 
 const log = require('../log')
+const { TOP_LEVEL_KEY } = require('../constants')
 const { truncateSpan, normalizeSpan } = require('./tags-processors')
 
 /**
  * Formats a span for JSON encoding.
  * @param {object} span - The span to format
+ * @param {boolean} isFirstSpan - Whether this is the first span in the trace
  * @returns {object} The formatted span
  */
-function formatSpan (span) {
+function formatSpan (span, isFirstSpan) {
   span = normalizeSpan(truncateSpan(span, false))
 
   if (span.span_events) {
@@ -16,6 +18,18 @@ function formatSpan (span) {
     delete span.span_events
   }
 
+  if (isFirstSpan) {
+    span.meta['_dd.compute_stats'] = '1'
+  }
+
+  if (span.parent_id?.toString(10) === '0') {
+    span.metrics._trace_root = 1
+  }
+
+  if (span.metrics[TOP_LEVEL_KEY]) {
+    span.metrics._top_level = 1
+  }
+
   return span
 }
 
@@ -83,7 +97,7 @@ class AgentlessJSONEncoder {
   encode (trace) {
     for (const span of trace) {
       try {
-        const formattedSpan = formatSpan(span)
+        const formattedSpan = formatSpan(span, this._spanCount === 0)
         const jsonSpan = spanToJSON(formattedSpan)
 
         this._spans.push(jsonSpan)

package/packages/dd-trace/src/exporters/agent/writer.js

@@ -1,7 +1,8 @@
 'use strict'
 
+const { inspect } = require('node:util')
 const request = require('../common/request')
-const { startupLog } = require('../../startup-log')
+const { logIntegrations, logAgentError } = require('../../startup-log')
 const runtimeMetrics = require('../../runtime_metrics')
 const log = require('../../log')
 const tracerVersion = require('../../../../../package.json').version
@@ -28,7 +29,7 @@ class AgentWriter extends BaseWriter {
     runtimeMetrics.increment(`${METRIC_PREFIX}.requests`, true)
 
     const { _headers, _lookup, _protocolVersion, _url } = this
-    makeRequest(_protocolVersion, data, count, _url, _headers, _lookup, true, (err, res, status, headers) => {
+    makeRequest(_protocolVersion, data, count, _url, _headers, _lookup, (err, res, status, headers) => {
       if (status) {
         runtimeMetrics.increment(`${METRIC_PREFIX}.responses`, true)
         runtimeMetrics.increment(`${METRIC_PREFIX}.responses.by.status`, `status:${status}`, true)
@@ -78,7 +79,7 @@ function getEncoder (protocolVersion) {
     : require('../../encode/0.4').AgentEncoder
 }
 
-function makeRequest (version, data, count, url, headers, lookup, needsStartupLog, cb) {
+function makeRequest (version, data, count, url, headers, lookup, cb) {
   const options = {
     path: `/v${version}/traces`,
     method: 'PUT',
@@ -98,11 +99,9 @@ function makeRequest (version, data, count, url, headers, lookup, needsStartupLo
   log.debug('Request to the agent: %j', options)
 
   request(data, options, (err, res, status, headers) => {
-    if (needsStartupLog) {
-      // Note that logging will only happen once, regardless of how many times this is called.
-      startupLog({
-        agentError: status !== 404 && status !== 200 ? err : undefined,
-      })
+    logIntegrations()
+    if (status !== 404 && status !== 200 && err) {
+      logAgentError({ status, message: err.message ?? inspect(err) })
     }
     cb(err, res, status, headers)
   })

package/packages/dd-trace/src/pkg.js

@@ -23,7 +23,7 @@ function findPkg () {
   if (filePath === undefined) return {}
 
   try {
-    return require(filePath)
+    return JSON.parse(fs.readFileSync(filePath, 'utf8'))
   } catch {
     return {}
   }

package/packages/dd-trace/src/proxy.js

@@ -6,7 +6,7 @@ const DatadogTracer = require('./tracer')
 const getConfig = require('./config')
 const runtimeMetrics = require('./runtime_metrics')
 const log = require('./log')
-const { setStartupLogPluginManager } = require('./startup-log')
+const { setStartupLogPluginManager, startupLog } = require('./startup-log')
 const DynamicInstrumentation = require('./debugger')
 const telemetry = require('./telemetry')
 const nomenclature = require('./service-naming')
@@ -292,6 +292,7 @@ class Tracer extends NoopProxy {
       this._pluginManager.configure(config)
       DynamicInstrumentation.configure(config)
       setStartupLogPluginManager(this._pluginManager)
+      startupLog()
     }
   }
 

package/packages/dd-trace/src/startup-log.js

@@ -11,41 +11,66 @@ let config
 let pluginManager
 /** @type {import('./sampling_rule')[]} */
 let samplingRules = []
-let alreadyRan = false
+let configAlreadyRan = false
+let integrationsAlreadyRan = false
+let agentErrorAlreadyRan = false
 
 /**
- * @param {{ status: number, message: string } } [agentError]
+ * Logs DATADOG TRACER CONFIGURATION immediately at init time.
+ * Excludes integrations_loaded since plugins haven't loaded yet.
  */
-function startupLog (agentError) {
-  if (alreadyRan || !config || !config.startupLogs || !pluginManager) {
+function startupLog () {
+  if (configAlreadyRan || !config || !config.startupLogs) {
     return
   }
 
-  alreadyRan = true
+  configAlreadyRan = true
 
-  const out = tracerInfo()
+  const out = configInfo()
 
-  if (agentError) {
-    out.agent_error = agentError.message
+  warn('DATADOG TRACER CONFIGURATION - ' + out)
+}
+
+/**
+ * Logs loaded integrations. Called from writer.js on first agent payload,
+ * by which time the app has loaded its dependencies.
+ */
+function logIntegrations () {
+  if (integrationsAlreadyRan || !config || !config.startupLogs || !pluginManager) {
+    return
   }
 
-  warn('DATADOG TRACER CONFIGURATION - ' + out)
-  if (agentError) {
-    warn('DATADOG TRACER DIAGNOSTIC - Agent Error: ' + agentError.message)
-    errors.agentError = {
-      code: agentError.status,
-      message: `Agent Error: ${agentError.message}`,
-    }
+  integrationsAlreadyRan = true
+
+  warn('DATADOG TRACER INTEGRATIONS LOADED - ' + JSON.stringify(Object.keys(pluginManager._pluginsByName)))
+}
+
+/**
+ * Logs agent error diagnostic.
+ * @param {{ status: number, message: string }} agentError
+ */
+function logAgentError (agentError) {
+  if (agentErrorAlreadyRan || !config || !config.startupLogs) {
+    return
+  }
+
+  agentErrorAlreadyRan = true
+
+  warn('DATADOG TRACER DIAGNOSTIC - Agent Error: ' + agentError.message)
+  errors.agentError = {
+    code: agentError.status,
+    message: `Agent Error: ${agentError.message}`,
   }
 }
 
 /**
+ * Returns config info without integrations (used by startupLog).
  * @returns {Record<string, unknown>}
  */
-function tracerInfo () {
+function configInfo () {
   const url = getAgentUrl(config)
 
-  const out = {
+  return {
     [inspect.custom] () {
       return String(this)
     },
@@ -73,11 +98,18 @@ function tracerInfo () {
     log_injection_enabled: !!config.logInjection,
     runtime_metrics_enabled: !!config.runtimeMetrics,
     profiling_enabled: config.profiling?.enabled === 'true' || config.profiling?.enabled === 'auto',
-    integrations_loaded: Object.keys(pluginManager._pluginsByName),
     appsec_enabled: config.appsec.enabled,
     data_streams_enabled: !!config.dsmEnabled,
   }
+}
 
+/**
+ * Returns full tracer info including integrations (used by flare module).
+ * @returns {Record<string, unknown>}
+ */
+function tracerInfo () {
+  const out = configInfo()
+  out.integrations_loaded = Object.keys(pluginManager._pluginsByName)
   return out
 }
 
@@ -104,6 +136,8 @@ function setSamplingRules (theRules) {
 
 module.exports = {
   startupLog,
+  logIntegrations,
+  logAgentError,
   setStartupLogConfig,
   setStartupLogPluginManager,
   setSamplingRules,