dd-trace 5.24.0 → 5.26.0

package/packages/datadog-instrumentations/src/kafkajs.js

@@ -52,45 +52,59 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf
     const send = producer.send
     const bootstrapServers = this._brokers
 
-    producer.send = function () {
-      const innerAsyncResource = new AsyncResource('bound-anonymous-fn')
+    const kafkaClusterIdPromise = getKafkaClusterId(this)
 
-      return innerAsyncResource.runInAsyncScope(() => {
-        if (!producerStartCh.hasSubscribers) {
-          return send.apply(this, arguments)
-        }
+    producer.send = function () {
+      const wrappedSend = (clusterId) => {
+        const innerAsyncResource = new AsyncResource('bound-anonymous-fn')
 
-        try {
-          const { topic, messages = [] } = arguments[0]
-          for (const message of messages) {
-            if (message !== null && typeof message === 'object') {
-              message.headers = message.headers || {}
-            }
+        return innerAsyncResource.runInAsyncScope(() => {
+          if (!producerStartCh.hasSubscribers) {
+            return send.apply(this, arguments)
           }
-          producerStartCh.publish({ topic, messages, bootstrapServers })
-
-          const result = send.apply(this, arguments)
-
-          result.then(
-            innerAsyncResource.bind(res => {
-              producerFinishCh.publish(undefined)
-              producerCommitCh.publish(res)
-            }),
-            innerAsyncResource.bind(err => {
-              if (err) {
-                producerErrorCh.publish(err)
+
+          try {
+            const { topic, messages = [] } = arguments[0]
+            for (const message of messages) {
+              if (message !== null && typeof message === 'object') {
+                message.headers = message.headers || {}
               }
-              producerFinishCh.publish(undefined)
-            })
-          )
+            }
+            producerStartCh.publish({ topic, messages, bootstrapServers, clusterId })
 
-          return result
-        } catch (e) {
-          producerErrorCh.publish(e)
-          producerFinishCh.publish(undefined)
-          throw e
-        }
-      })
+            const result = send.apply(this, arguments)
+
+            result.then(
+              innerAsyncResource.bind(res => {
+                producerFinishCh.publish(undefined)
+                producerCommitCh.publish(res)
+              }),
+              innerAsyncResource.bind(err => {
+                if (err) {
+                  producerErrorCh.publish(err)
+                }
+                producerFinishCh.publish(undefined)
+              })
+            )
+
+            return result
+          } catch (e) {
+            producerErrorCh.publish(e)
+            producerFinishCh.publish(undefined)
+            throw e
+          }
+        })
+      }
+
+      if (!isPromise(kafkaClusterIdPromise)) {
+        // promise is already resolved
+        return wrappedSend(kafkaClusterIdPromise)
+      } else {
+        // promise is not resolved
+        return kafkaClusterIdPromise.then((clusterId) => {
+          return wrappedSend(clusterId)
+        })
+      }
     }
     return producer
   })
@@ -100,15 +114,17 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf
       return createConsumer.apply(this, arguments)
     }
 
-    const eachMessageExtractor = (args) => {
+    const kafkaClusterIdPromise = getKafkaClusterId(this)
+
+    const eachMessageExtractor = (args, clusterId) => {
       const { topic, partition, message } = args[0]
-      return { topic, partition, message, groupId }
+      return { topic, partition, message, groupId, clusterId }
     }
 
-    const eachBatchExtractor = (args) => {
+    const eachBatchExtractor = (args, clusterId) => {
       const { batch } = args[0]
       const { topic, partition, messages } = batch
-      return { topic, partition, messages, groupId }
+      return { topic, partition, messages, groupId, clusterId }
     }
 
     const consumer = createConsumer.apply(this, arguments)
@@ -116,43 +132,53 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf
     consumer.on(consumer.events.COMMIT_OFFSETS, commitsFromEvent)
 
     const run = consumer.run
-
     const groupId = arguments[0].groupId
+
     consumer.run = function ({ eachMessage, eachBatch, ...runArgs }) {
-      eachMessage = wrapFunction(
-        eachMessage,
-        consumerStartCh,
-        consumerFinishCh,
-        consumerErrorCh,
-        eachMessageExtractor
-      )
-
-      eachBatch = wrapFunction(
-        eachBatch,
-        batchConsumerStartCh,
-        batchConsumerFinishCh,
-        batchConsumerErrorCh,
-        eachBatchExtractor
-      )
-
-      return run({
-        eachMessage,
-        eachBatch,
-        ...runArgs
-      })
+      const wrapConsume = (clusterId) => {
+        return run({
+          eachMessage: wrappedCallback(
+            eachMessage,
+            consumerStartCh,
+            consumerFinishCh,
+            consumerErrorCh,
+            eachMessageExtractor,
+            clusterId
+          ),
+          eachBatch: wrappedCallback(
+            eachBatch,
+            batchConsumerStartCh,
+            batchConsumerFinishCh,
+            batchConsumerErrorCh,
+            eachBatchExtractor,
+            clusterId
+          ),
+          ...runArgs
+        })
+      }
+
+      if (!isPromise(kafkaClusterIdPromise)) {
+        // promise is already resolved
+        return wrapConsume(kafkaClusterIdPromise)
+      } else {
+        // promise is not resolved
+        return kafkaClusterIdPromise.then((clusterId) => {
+          return wrapConsume(clusterId)
+        })
+      }
     }
-
     return consumer
   })
   return Kafka
 })
 
-const wrapFunction = (fn, startCh, finishCh, errorCh, extractArgs) => {
+const wrappedCallback = (fn, startCh, finishCh, errorCh, extractArgs, clusterId) => {
   return typeof fn === 'function'
     ? function (...args) {
       const innerAsyncResource = new AsyncResource('bound-anonymous-fn')
       return innerAsyncResource.runInAsyncScope(() => {
-        const extractedArgs = extractArgs(args)
+        const extractedArgs = extractArgs(args, clusterId)
+
         startCh.publish(extractedArgs)
         try {
           const result = fn.apply(this, args)
@@ -179,3 +205,37 @@ const wrapFunction = (fn, startCh, finishCh, errorCh, extractArgs) => {
     }
     : fn
 }
+
+const getKafkaClusterId = (kafka) => {
+  if (kafka._ddKafkaClusterId) {
+    return kafka._ddKafkaClusterId
+  }
+
+  if (!kafka.admin) {
+    return null
+  }
+
+  const admin = kafka.admin()
+
+  if (!admin.describeCluster) {
+    return null
+  }
+
+  return admin.connect()
+    .then(() => {
+      return admin.describeCluster()
+    })
+    .then((clusterInfo) => {
+      const clusterId = clusterInfo?.clusterId
+      kafka._ddKafkaClusterId = clusterId
+      admin.disconnect()
+      return clusterId
+    })
+    .catch((error) => {
+      throw error
+    })
+}
+
+function isPromise (obj) {
+  return !!obj && (typeof obj === 'object' || typeof obj === 'function') && typeof obj.then === 'function'
+}

package/packages/datadog-instrumentations/src/mocha/utils.js

@@ -280,12 +280,12 @@ function getOnFailHandler (isMain) {
 }
 
 function getOnTestRetryHandler () {
-  return function (test) {
+  return function (test, err) {
     const asyncResource = getTestAsyncResource(test)
     if (asyncResource) {
       const isFirstAttempt = test._currentRetry === 0
       asyncResource.runInAsyncScope(() => {
-        testRetryCh.publish(isFirstAttempt)
+        testRetryCh.publish({ isFirstAttempt, err })
       })
     }
     const key = getTestToArKey(test)

package/packages/datadog-instrumentations/src/multer.js

@@ -0,0 +1,37 @@
+'use strict'
+
+const shimmer = require('../../datadog-shimmer')
+const { channel, addHook, AsyncResource } = require('./helpers/instrument')
+
+const multerReadCh = channel('datadog:multer:read:finish')
+
+function publishRequestBodyAndNext (req, res, next) {
+  return shimmer.wrapFunction(next, next => function () {
+    if (multerReadCh.hasSubscribers && req) {
+      const abortController = new AbortController()
+      const body = req.body
+
+      multerReadCh.publish({ req, res, body, abortController })
+
+      if (abortController.signal.aborted) return
+    }
+
+    return next.apply(this, arguments)
+  })
+}
+
+addHook({
+  name: 'multer',
+  file: 'lib/make-middleware.js',
+  versions: ['^1.4.4-lts.1']
+}, makeMiddleware => {
+  return shimmer.wrapFunction(makeMiddleware, makeMiddleware => function () {
+    const middleware = makeMiddleware.apply(this, arguments)
+
+    return shimmer.wrapFunction(middleware, middleware => function wrapMulterMiddleware (req, res, next) {
+      const nextResource = new AsyncResource('bound-anonymous-fn')
+      arguments[2] = nextResource.bind(publishRequestBodyAndNext(req, res, next))
+      return middleware.apply(this, arguments)
+    })
+  })
+})

package/packages/datadog-instrumentations/src/openai.js

@@ -3,8 +3,8 @@
 const { addHook } = require('./helpers/instrument')
 const shimmer = require('../../datadog-shimmer')
 
-const tracingChannel = require('dc-polyfill').tracingChannel
-const ch = tracingChannel('apm:openai:request')
+const dc = require('dc-polyfill')
+const ch = dc.tracingChannel('apm:openai:request')
 
 const V4_PACKAGE_SHIMS = [
   {

package/packages/datadog-instrumentations/src/pug.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const shimmer = require('../../datadog-shimmer')
+const { channel, addHook } = require('./helpers/instrument')
+
+const pugCompileCh = channel('datadog:pug:compile:start')
+
+function wrapCompile (compile) {
+  return function wrappedCompile (source) {
+    if (pugCompileCh.hasSubscribers) {
+      pugCompileCh.publish({ source })
+    }
+
+    return compile.apply(this, arguments)
+  }
+}
+
+addHook({ name: 'pug', versions: ['>=2.0.4'] }, compiler => {
+  shimmer.wrap(compiler, 'compile', wrapCompile)
+  shimmer.wrap(compiler, 'compileClientWithDependenciesTracked', wrapCompile)
+
+  return compiler
+})

package/packages/datadog-instrumentations/src/router.js

@@ -112,7 +112,6 @@ function createWrapRouterMethod (name) {
       path: pattern instanceof RegExp ? `(${pattern})` : pattern,
       test: layer => {
         const matchers = layerMatchers.get(layer)
-
         return !isFastStar(layer, matchers) &&
           !isFastSlash(layer, matchers) &&
           cachedPathToRegExp(pattern).test(layer.path)
@@ -121,7 +120,7 @@ function createWrapRouterMethod (name) {
   }
 
   function isFastStar (layer, matchers) {
-    if (layer.regexp.fast_star !== undefined) {
+    if (layer.regexp?.fast_star !== undefined) {
       return layer.regexp.fast_star
     }
 
@@ -129,7 +128,7 @@ function createWrapRouterMethod (name) {
   }
 
   function isFastSlash (layer, matchers) {
-    if (layer.regexp.fast_slash !== undefined) {
+    if (layer.regexp?.fast_slash !== undefined) {
       return layer.regexp.fast_slash
     }
 

package/packages/datadog-instrumentations/src/url.js

@@ -0,0 +1,84 @@
+'use strict'
+
+const { addHook, channel } = require('./helpers/instrument')
+const shimmer = require('../../datadog-shimmer')
+const names = ['url', 'node:url']
+
+const parseFinishedChannel = channel('datadog:url:parse:finish')
+const urlGetterChannel = channel('datadog:url:getter:finish')
+const instrumentedGetters = ['host', 'origin', 'hostname']
+
+addHook({ name: names }, function (url) {
+  shimmer.wrap(url, 'parse', (parse) => {
+    return function wrappedParse (input) {
+      const parsedValue = parse.apply(this, arguments)
+      if (!parseFinishedChannel.hasSubscribers) return parsedValue
+
+      parseFinishedChannel.publish({
+        input,
+        parsed: parsedValue,
+        isURL: false
+      })
+
+      return parsedValue
+    }
+  })
+
+  const URLPrototype = url.URL.prototype.constructor.prototype
+  instrumentedGetters.forEach(property => {
+    const originalDescriptor = Object.getOwnPropertyDescriptor(URLPrototype, property)
+
+    if (originalDescriptor?.get) {
+      const newDescriptor = shimmer.wrap(originalDescriptor, 'get', function (originalGet) {
+        return function get () {
+          const result = originalGet.apply(this, arguments)
+          if (!urlGetterChannel.hasSubscribers) return result
+
+          const context = { urlObject: this, result, property }
+          urlGetterChannel.publish(context)
+
+          return context.result
+        }
+      })
+
+      Object.defineProperty(URLPrototype, property, newDescriptor)
+    }
+  })
+
+  shimmer.wrap(url, 'URL', (URL) => {
+    return class extends URL {
+      constructor (input, base) {
+        super(...arguments)
+
+        if (!parseFinishedChannel.hasSubscribers) return
+
+        parseFinishedChannel.publish({
+          input,
+          base,
+          parsed: this,
+          isURL: true
+        })
+      }
+    }
+  })
+
+  if (url.URL.parse) {
+    shimmer.wrap(url.URL, 'parse', (parse) => {
+      return function wrappedParse (input, base) {
+        const parsedValue = parse.apply(this, arguments)
+        if (!parseFinishedChannel.hasSubscribers) return parsedValue
+
+        parseFinishedChannel.publish({
+          input,
+          base,
+          parsed: parsedValue,
+          isURL: true
+        })
+
+        return parsedValue
+      }
+    })
+  }
+
+  return url
+})

package/packages/datadog-instrumentations/src/utils/src/extract-package-and-module-path.js

@@ -6,7 +6,7 @@ const NM = 'node_modules/'
  * For a given full path to a module,
  *   return the package name it belongs to and the local path to the module
  *   input: '/foo/node_modules/@co/stuff/foo/bar/baz.js'
- *   output: { pkg: '@co/stuff', path: 'foo/bar/baz.js' }
+ *   output: { pkg: '@co/stuff', path: 'foo/bar/baz.js',  pkgJson: '/foo/node_modules/@co/stuff/package.json' }
  */
 module.exports = function extractPackageAndModulePath (fullPath) {
   const nm = fullPath.lastIndexOf(NM)
@@ -17,17 +17,20 @@ module.exports = function extractPackageAndModulePath (fullPath) {
   const subPath = fullPath.substring(nm + NM.length)
   const firstSlash = subPath.indexOf('/')
 
+  const firstPath = fullPath.substring(fullPath[0], nm + NM.length)
+
   if (subPath[0] === '@') {
     const secondSlash = subPath.substring(firstSlash + 1).indexOf('/')
-
     return {
       pkg: subPath.substring(0, firstSlash + 1 + secondSlash),
-      path: subPath.substring(firstSlash + 1 + secondSlash + 1)
+      path: subPath.substring(firstSlash + 1 + secondSlash + 1),
+      pkgJson: firstPath + subPath.substring(0, firstSlash + 1 + secondSlash) + '/package.json'
     }
   }
 
   return {
     pkg: subPath.substring(0, firstSlash),
-    path: subPath.substring(firstSlash + 1)
+    path: subPath.substring(firstSlash + 1),
+    pkgJson: firstPath + subPath.substring(0, firstSlash) + '/package.json'
   }
 }

package/packages/datadog-plugin-amqplib/src/consumer.js

@@ -9,33 +9,34 @@ class AmqplibConsumerPlugin extends ConsumerPlugin {
   static get id () { return 'amqplib' }
   static get operation () { return 'command' }
 
-  start ({ method, fields, message }) {
+  start ({ method, fields, message, queue }) {
     if (method !== 'basic.deliver' && method !== 'basic.get') return
 
     const childOf = extract(this.tracer, message)
 
+    const queueName = queue || fields.queue || fields.routingKey
     const span = this.startSpan({
       childOf,
       resource: getResourceName(method, fields),
       type: 'worker',
       meta: {
-        'amqp.queue': fields.queue,
+        'amqp.queue': queueName,
         'amqp.exchange': fields.exchange,
         'amqp.routingKey': fields.routingKey,
         'amqp.consumerTag': fields.consumerTag,
         'amqp.source': fields.source,
         'amqp.destination': fields.destination
-      }
+      },
+      extractedLinks: childOf?._links
     })
 
     if (
       this.config.dsmEnabled && message?.properties?.headers
     ) {
       const payloadSize = getAmqpMessageSize({ headers: message.properties.headers, content: message.content })
-      const queue = fields.queue ? fields.queue : fields.routingKey
       this.tracer.decodeDataStreamsContext(message.properties.headers)
       this.tracer
-        .setCheckpoint(['direction:in', `topic:${queue}`, 'type:rabbitmq'], span, payloadSize)
+        .setCheckpoint(['direction:in', `topic:${queueName}`, 'type:rabbitmq'], span, payloadSize)
     }
   }
 }

package/packages/datadog-plugin-aws-sdk/src/base.js

@@ -93,6 +93,7 @@ class BaseAwsSdkPlugin extends ClientPlugin {
         this.responseExtractDSMContext(operation, params, response.data ?? response, span)
       }
       this.addResponseTags(span, response)
+      this.addSpanPointers(span, response)
       this.finish(span, response, response.error)
     })
   }
@@ -101,6 +102,10 @@ class BaseAwsSdkPlugin extends ClientPlugin {
     // implemented by subclasses, or not
   }
 
+  addSpanPointers (span, response) {
+    // Optionally implemented by subclasses, for services where we're unable to inject trace context
+  }
+
   operationFromRequest (request) {
     // can be overriden by subclasses
     return this.operationName({

package/packages/datadog-plugin-aws-sdk/src/services/eventbridge.js

@@ -4,6 +4,7 @@ const BaseAwsSdkPlugin = require('../base')
 
 class EventBridge extends BaseAwsSdkPlugin {
   static get id () { return 'eventbridge' }
+  static get isPayloadReporter () { return true }
 
   generateTags (params, operation, response) {
     if (!params || !params.source) return {}

package/packages/datadog-plugin-aws-sdk/src/services/kinesis.js

@@ -10,6 +10,7 @@ const { storage } = require('../../../datadog-core')
 class Kinesis extends BaseAwsSdkPlugin {
   static get id () { return 'kinesis' }
   static get peerServicePrecursors () { return ['streamname'] }
+  static get isPayloadReporter () { return true }
 
   constructor (...args) {
     super(...args)
@@ -41,7 +42,8 @@ class Kinesis extends BaseAwsSdkPlugin {
               {},
               this.requestTags.get(request) || {},
               { 'span.kind': 'server' }
-            )
+            ),
+            extractedLinks: responseExtraction.maybeChildOf._links
           }
           span = plugin.tracer.startSpan('aws.response', options)
           this.enter(span, store)
@@ -112,14 +114,15 @@ class Kinesis extends BaseAwsSdkPlugin {
     response.Records.forEach(record => {
       const parsedAttributes = JSON.parse(Buffer.from(record.Data).toString())
 
-      if (
-        parsedAttributes?._datadog && streamName
-      ) {
-        const payloadSize = getSizeOrZero(record.Data)
+      const payloadSize = getSizeOrZero(record.Data)
+      if (parsedAttributes?._datadog) {
         this.tracer.decodeDataStreamsContext(parsedAttributes._datadog)
-        this.tracer
-          .setCheckpoint(['direction:in', `topic:${streamName}`, 'type:kinesis'], span, payloadSize)
       }
+      const tags = streamName
+        ? ['direction:in', `topic:${streamName}`, 'type:kinesis']
+        : ['direction:in', 'type:kinesis']
+      this.tracer
+        .setCheckpoint(tags, span, payloadSize)
     })
   }
 

package/packages/datadog-plugin-aws-sdk/src/services/s3.js

@@ -1,10 +1,14 @@
 'use strict'
 
 const BaseAwsSdkPlugin = require('../base')
+const log = require('../../../dd-trace/src/log')
+const { generatePointerHash } = require('../../../dd-trace/src/util')
+const { S3_PTR_KIND, SPAN_POINTER_DIRECTION } = require('../../../dd-trace/src/constants')
 
 class S3 extends BaseAwsSdkPlugin {
   static get id () { return 's3' }
   static get peerServicePrecursors () { return ['bucketname'] }
+  static get isPayloadReporter () { return true }
 
   generateTags (params, operation, response) {
     const tags = {}
@@ -17,6 +21,37 @@ class S3 extends BaseAwsSdkPlugin {
       bucketname: params.Bucket
     })
   }
+
+  addSpanPointers (span, response) {
+    const request = response?.request
+    const operationName = request?.operation
+    if (!['putObject', 'copyObject', 'completeMultipartUpload'].includes(operationName)) {
+      // We don't create span links for other S3 operations.
+      return
+    }
+
+    // AWS v2: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html
+    // AWS v3: https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/s3/
+    const bucketName = request?.params?.Bucket
+    const objectKey = request?.params?.Key
+    let eTag =
+      response?.ETag || // v3 PutObject & CompleteMultipartUpload
+      response?.CopyObjectResult?.ETag || // v3 CopyObject
+      response?.data?.ETag || // v2 PutObject & CompleteMultipartUpload
+      response?.data?.CopyObjectResult?.ETag // v2 CopyObject
+
+    if (!bucketName || !objectKey || !eTag) {
+      log.debug('Unable to calculate span pointer hash because of missing parameters.')
+      return
+    }
+
+    // https://github.com/DataDog/dd-span-pointer-rules/blob/main/AWS/S3/Object/README.md
+    if (eTag.startsWith('"') && eTag.endsWith('"')) {
+      eTag = eTag.slice(1, -1)
+    }
+    const pointerHash = generatePointerHash([bucketName, objectKey, eTag])
+    span.addSpanPointer(S3_PTR_KIND, SPAN_POINTER_DIRECTION.DOWNSTREAM, pointerHash)
+  }
 }
 
 module.exports = S3

package/packages/datadog-plugin-aws-sdk/src/services/sqs.js

@@ -9,6 +9,7 @@ const { DsmPathwayCodec } = require('../../../dd-trace/src/datastreams/pathway')
 class Sqs extends BaseAwsSdkPlugin {
   static get id () { return 'sqs' }
   static get peerServicePrecursors () { return ['queuename'] }
+  static get isPayloadReporter () { return true }
 
   constructor (...args) {
     super(...args)
@@ -32,7 +33,8 @@ class Sqs extends BaseAwsSdkPlugin {
             {},
             this.requestTags.get(request) || {},
             { 'span.kind': 'server' }
-          )
+          ),
+          extractedLinks: contextExtraction.datadogContext._links
         }
         parsedMessageAttributes = contextExtraction.parsedAttributes
         span = plugin.tracer.startSpan('aws.response', options)
@@ -41,7 +43,7 @@ class Sqs extends BaseAwsSdkPlugin {
       // extract DSM context after as we might not have a parent-child but may have a DSM context
 
       this.responseExtractDSMContext(
-        request.operation, request.params, response, span || null, { parsedMessageAttributes }
+        request.operation, request.params, response, span || null, { parsedAttributes: parsedMessageAttributes }
       )
     })
 
@@ -194,16 +196,16 @@ class Sqs extends BaseAwsSdkPlugin {
           parsedAttributes = this.parseDatadogAttributes(message.MessageAttributes._datadog)
         }
       }
+      const payloadSize = getHeadersSize({
+        Body: message.Body,
+        MessageAttributes: message.MessageAttributes
+      })
+      const queue = params.QueueUrl.split('/').pop()
       if (parsedAttributes) {
-        const payloadSize = getHeadersSize({
-          Body: message.Body,
-          MessageAttributes: message.MessageAttributes
-        })
-        const queue = params.QueueUrl.split('/').pop()
         this.tracer.decodeDataStreamsContext(parsedAttributes)
-        this.tracer
-          .setCheckpoint(['direction:in', `topic:${queue}`, 'type:sqs'], span, payloadSize)
       }
+      this.tracer
+        .setCheckpoint(['direction:in', `topic:${queue}`, 'type:sqs'], span, payloadSize)
     })
   }