dd-trace 5.1.0 → 5.2.0

package/LICENSE-3rdparty.csv

@@ -17,10 +17,7 @@ require,istanbul-lib-coverage,BSD-3-Clause,Copyright 2012-2015 Yahoo! Inc.
 require,jest-docblock,MIT,Copyright Meta Platforms, Inc. and affiliates.
 require,koalas,MIT,Copyright 2013-2017 Brian Woodward
 require,limiter,MIT,Copyright 2011 John Hurliman
-require,lodash.kebabcase,MIT,Copyright JS Foundation and other contributors
-require,lodash.pick,MIT,Copyright JS Foundation and other contributors
 require,lodash.sortby,MIT,Copyright JS Foundation and other contributors
-require,lodash.uniq,MIT,Copyright JS Foundation and other contributors
 require,lru-cache,ISC,Copyright (c) 2010-2022 Isaac Z. Schlueter and Contributors
 require,methods,MIT,Copyright 2013-2014 TJ Holowaychuk
 require,module-details-from-path,MIT,Copyright 2016 Thomas Watson Steen

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "5.1.0",
+  "version": "5.2.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -87,10 +87,7 @@
     "jest-docblock": "^29.7.0",
     "koalas": "^1.0.2",
     "limiter": "1.1.5",
-    "lodash.kebabcase": "^4.1.1",
-    "lodash.pick": "^4.4.0",
     "lodash.sortby": "^4.7.0",
-    "lodash.uniq": "^4.5.0",
     "lru-cache": "^7.14.0",
     "methods": "^1.1.2",
     "module-details-from-path": "^1.0.3",

package/packages/datadog-instrumentations/src/amqplib.js

@@ -5,7 +5,7 @@ const {
   addHook,
   AsyncResource
 } = require('./helpers/instrument')
-const kebabCase = require('lodash.kebabcase')
+const kebabCase = require('../../utils/src/kebabcase')
 const shimmer = require('../../datadog-shimmer')
 
 const startCh = channel('apm:amqplib:command:start')

package/packages/datadog-instrumentations/src/helpers/hooks.js

@@ -73,6 +73,7 @@ module.exports = {
   'mongodb': () => require('../mongodb'),
   'mongodb-core': () => require('../mongodb-core'),
   'mongoose': () => require('../mongoose'),
+  'mquery': () => require('../mquery'),
   'mysql': () => require('../mysql'),
   'mysql2': () => require('../mysql2'),
   'net': () => require('../net'),

package/packages/datadog-instrumentations/src/mquery.js

@@ -0,0 +1,65 @@
+'use strict'
+
+const dc = require('dc-polyfill')
+const {
+  channel,
+  addHook
+} = require('./helpers/instrument')
+const shimmer = require('../../datadog-shimmer')
+
+const prepareCh = channel('datadog:mquery:filter:prepare')
+const tracingCh = dc.tracingChannel('datadog:mquery:filter')
+
+const methods = [
+  'find',
+  'findOne',
+  'findOneAndRemove',
+  'findOneAndDelete',
+  'count',
+  'distinct',
+  'where'
+]
+
+const methodsOptionalArgs = ['findOneAndUpdate']
+
+function getFilters (args, methodName) {
+  const [arg0, arg1] = args
+
+  const filters = arg0 && typeof arg0 === 'object' ? [arg0] : []
+
+  if (arg1 && typeof arg1 === 'object' && methodsOptionalArgs.includes(methodName)) {
+    filters.push(arg1)
+  }
+
+  return filters
+}
+
+addHook({
+  name: 'mquery',
+  versions: ['>=5.0.0']
+}, Query => {
+  [...methods, ...methodsOptionalArgs].forEach(methodName => {
+    if (!(methodName in Query.prototype)) return
+
+    shimmer.wrap(Query.prototype, methodName, method => {
+      return function () {
+        if (prepareCh.hasSubscribers) {
+          const filters = getFilters(arguments, methodName)
+          if (filters?.length) {
+            prepareCh.publish({ filters })
+          }
+        }
+
+        return method.apply(this, arguments)
+      }
+    })
+  })
+
+  shimmer.wrap(Query.prototype, 'exec', originalExec => {
+    return function wrappedExec () {
+      return tracingCh.tracePromise(originalExec, {}, this, arguments)
+    }
+  })
+
+  return Query
+})

package/packages/datadog-plugin-aws-sdk/src/services/kinesis.js

@@ -1,6 +1,11 @@
 'use strict'
+const {
+  CONTEXT_PROPAGATION_KEY
+} = require('../../../dd-trace/src/datastreams/processor')
+const { encodePathwayContext } = require('../../../dd-trace/src/datastreams/pathway')
 const log = require('../../../dd-trace/src/log')
 const BaseAwsSdkPlugin = require('../base')
+
 class Kinesis extends BaseAwsSdkPlugin {
   static get id () { return 'kinesis' }
   static get peerServicePrecursors () { return ['streamname'] }
@@ -37,8 +42,9 @@ class Kinesis extends BaseAwsSdkPlugin {
       if (!request.params) {
         return
       }
-
       const traceData = {}
+
+      // inject data with DD context
       this.tracer.inject(span, 'text_map', traceData)
       let injectPath
       if (request.params.Records && request.params.Records.length > 0) {
@@ -49,9 +55,30 @@ class Kinesis extends BaseAwsSdkPlugin {
         log.error('No valid payload passed, unable to pass trace context')
         return
       }
+
       const parsedData = this._tryParse(injectPath.Data)
       if (parsedData) {
         parsedData._datadog = traceData
+
+        // set DSM hash if enabled
+        if (this.config.dsmEnabled) {
+          // get payload size of request data
+          const payloadSize = Buffer.from(JSON.stringify(parsedData)).byteLength
+          let stream
+          // users can optionally use either stream name or stream arn
+          if (request.params && request.params.StreamArn) {
+            stream = request.params.StreamArn
+          } else if (request.params && request.params.StreamName) {
+            stream = request.params.StreamName
+          }
+          const dataStreamsContext = this.tracer
+            .setCheckpoint(['direction:out', `topic:${stream}`, 'type:kinesis'], span, payloadSize)
+          if (dataStreamsContext) {
+            const pathwayCtx = encodePathwayContext(dataStreamsContext)
+            parsedData._datadog[CONTEXT_PROPAGATION_KEY] = pathwayCtx.toJSON()
+          }
+        }
+
         const finalData = Buffer.from(JSON.stringify(parsedData))
         const byteSize = finalData.length
         // Kinesis max payload size is 1MB

package/packages/datadog-plugin-aws-sdk/src/services/sns.js

@@ -1,4 +1,6 @@
 'use strict'
+const { CONTEXT_PROPAGATION_KEY, getHeadersSize } = require('../../../dd-trace/src/datastreams/processor')
+const { encodePathwayContext } = require('../../../dd-trace/src/datastreams/pathway')
 const log = require('../../../dd-trace/src/log')
 const BaseAwsSdkPlugin = require('../base')
 
@@ -11,6 +13,7 @@ class Sns extends BaseAwsSdkPlugin {
 
     if (!params.TopicArn && !(response.data && response.data.TopicArn)) return {}
     const TopicArn = params.TopicArn || response.data.TopicArn
+
     // Split the ARN into its parts
     // ex.'arn:aws:sns:us-east-1:123456789012:my-topic'
     const arnParts = TopicArn.split(':')
@@ -72,10 +75,25 @@ class Sns extends BaseAwsSdkPlugin {
     }
     const ddInfo = {}
     this.tracer.inject(span, 'text_map', ddInfo)
+    // add ddInfo before checking DSM so we can include DD attributes in payload size
     params.MessageAttributes._datadog = {
       DataType: 'Binary',
-      BinaryValue: Buffer.from(JSON.stringify(ddInfo)) // BINARY types are automatically base64 encoded
+      BinaryValue: ddInfo
+    }
+    if (this.config.dsmEnabled) {
+      const payloadSize = getHeadersSize({
+        Message: params.Message,
+        MessageAttributes: params.MessageAttributes
+      })
+      const dataStreamsContext = this.tracer
+        .setCheckpoint(['direction:out', `topic:${params.TopicArn}`, 'type:sns'], span, payloadSize)
+      if (dataStreamsContext) {
+        const pathwayCtx = encodePathwayContext(dataStreamsContext)
+        ddInfo[CONTEXT_PROPAGATION_KEY] = pathwayCtx.toJSON()
+      }
     }
+    // BINARY types are automatically base64 encoded
+    params.MessageAttributes._datadog.BinaryValue = Buffer.from(JSON.stringify(ddInfo))
   }
 }
 

package/packages/datadog-plugin-aws-sdk/src/services/sqs.js

@@ -3,6 +3,8 @@
 const log = require('../../../dd-trace/src/log')
 const BaseAwsSdkPlugin = require('../base')
 const { storage } = require('../../../datadog-core')
+const { CONTEXT_PROPAGATION_KEY, getHeadersSize } = require('../../../dd-trace/src/datastreams/processor')
+const { encodePathwayContext } = require('../../../dd-trace/src/datastreams/pathway')
 
 class Sqs extends BaseAwsSdkPlugin {
   static get id () { return 'sqs' }
@@ -19,20 +21,27 @@ class Sqs extends BaseAwsSdkPlugin {
       const { request, response } = obj
       const store = storage.getStore()
       const plugin = this
-      const maybeChildOf = this.responseExtract(request.params, request.operation, response)
-      if (maybeChildOf) {
+      const contextExtraction = this.responseExtract(request.params, request.operation, response)
+      let span
+      let parsedMessageAttributes
+      if (contextExtraction && contextExtraction.datadogContext) {
         obj.needsFinish = true
         const options = {
-          childOf: maybeChildOf,
+          childOf: contextExtraction.datadogContext,
           tags: Object.assign(
             {},
             this.requestTags.get(request) || {},
             { 'span.kind': 'server' }
           )
         }
-        const span = plugin.tracer.startSpan('aws.response', options)
+        parsedMessageAttributes = contextExtraction.parsedAttributes
+        span = plugin.tracer.startSpan('aws.response', options)
         this.enter(span, store)
       }
+      // extract DSM context after as we might not have a parent-child but may have a DSM context
+      this.responseExtractDSMContext(
+        request.operation, request.params, response, span ?? null, parsedMessageAttributes ?? null
+      )
     })
 
     this.addSub('apm:aws:response:finish:sqs', err => {
@@ -133,19 +142,69 @@ class Sqs extends BaseAwsSdkPlugin {
 
     const datadogAttribute = message.MessageAttributes._datadog
 
+    const parsedAttributes = this.parseDatadogAttributes(datadogAttribute)
+    if (parsedAttributes) {
+      return {
+        datadogContext: this.tracer.extract('text_map', parsedAttributes),
+        parsedAttributes: parsedAttributes
+      }
+    }
+  }
+
+  parseDatadogAttributes (attributes) {
     try {
-      if (datadogAttribute.StringValue) {
-        const textMap = datadogAttribute.StringValue
-        return this.tracer.extract('text_map', JSON.parse(textMap))
-      } else if (datadogAttribute.Type === 'Binary') {
-        const buffer = Buffer.from(datadogAttribute.Value, 'base64')
-        return this.tracer.extract('text_map', JSON.parse(buffer))
+      if (attributes.StringValue) {
+        const textMap = attributes.StringValue
+        return JSON.parse(textMap)
+      } else if (attributes.Type === 'Binary') {
+        const buffer = Buffer.from(attributes.Value, 'base64')
+        return JSON.parse(buffer)
       }
     } catch (e) {
       log.error(e)
     }
   }
 
+  responseExtractDSMContext (operation, params, response, span, parsedAttributes) {
+    if (!this.config.dsmEnabled) return
+    if (operation !== 'receiveMessage') return
+    if (!response || !response.Messages || !response.Messages[0]) return
+
+    // we only want to set the payloadSize on the span if we have one message
+    span = response.Messages.length > 1 ? null : span
+
+    response.Messages.forEach(message => {
+      // we may have already parsed the message attributes when extracting trace context
+      if (!parsedAttributes) {
+        if (message.Body) {
+          try {
+            const body = JSON.parse(message.Body)
+
+            // SNS to SQS
+            if (body.Type === 'Notification') {
+              message = body
+            }
+          } catch (e) {
+            // SQS to SQS
+          }
+        }
+        if (message.MessageAttributes && message.MessageAttributes._datadog) {
+          parsedAttributes = this.parseDatadogAttributes(message.MessageAttributes._datadog)
+        }
+      }
+      if (parsedAttributes && parsedAttributes[CONTEXT_PROPAGATION_KEY]) {
+        const payloadSize = getHeadersSize({
+          Body: message.Body,
+          MessageAttributes: message.MessageAttributes
+        })
+        const queue = params.QueueUrl.split('/').pop()
+        this.tracer.decodeDataStreamsContext(Buffer.from(parsedAttributes[CONTEXT_PROPAGATION_KEY]))
+        this.tracer
+          .setCheckpoint(['direction:in', `topic:${queue}`, 'type:sqs'], span, payloadSize)
+      }
+    })
+  }
+
   requestInject (span, request) {
     const operation = request.operation
     if (operation === 'sendMessage') {
@@ -164,6 +223,20 @@ class Sqs extends BaseAwsSdkPlugin {
         DataType: 'String',
         StringValue: JSON.stringify(ddInfo)
       }
+      if (this.config.dsmEnabled) {
+        const payloadSize = getHeadersSize({
+          Body: request.params.MessageBody,
+          MessageAttributes: request.params.MessageAttributes
+        })
+        const queue = request.params.QueueUrl.split('/').pop()
+        const dataStreamsContext = this.tracer
+          .setCheckpoint(['direction:out', `topic:${queue}`, 'type:sqs'], span, payloadSize)
+        if (dataStreamsContext) {
+          const pathwayCtx = encodePathwayContext(dataStreamsContext)
+          ddInfo[CONTEXT_PROPAGATION_KEY] = pathwayCtx.toJSON()
+        }
+      }
+      request.params.MessageAttributes._datadog.StringValue = JSON.stringify(ddInfo)
     }
   }
 }

package/packages/datadog-plugin-graphql/src/index.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const pick = require('../../utils/src/pick')
 const CompositePlugin = require('../../dd-trace/src/plugins/composite')
 const log = require('../../dd-trace/src/log')
 const GraphQLExecutePlugin = require('./execute')
@@ -63,10 +64,4 @@ function getHooks (config) {
   return { execute, parse, validate }
 }
 
-// non-lodash pick
-
-function pick (obj, selectors) {
-  return Object.fromEntries(Object.entries(obj).filter(([key]) => selectors.includes(key)))
-}
-
 module.exports = GraphQLPlugin

package/packages/datadog-plugin-grpc/src/util.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const pick = require('lodash.pick')
+const pick = require('../../utils/src/pick')
 const log = require('../../dd-trace/src/log')
 
 module.exports = {

package/packages/dd-trace/src/appsec/iast/analyzers/nosql-injection-mongodb-analyzer.js

@@ -9,7 +9,7 @@ const { storage } = require('../../../../../datadog-core')
 const { getIastContext } = require('../iast-context')
 const { HTTP_REQUEST_PARAMETER, HTTP_REQUEST_BODY } = require('../taint-tracking/source-types')
 
-const EXCLUDED_PATHS_FROM_STACK = getNodeModulesPaths('mongodb', 'mongoose')
+const EXCLUDED_PATHS_FROM_STACK = getNodeModulesPaths('mongodb', 'mongoose', 'mquery')
 const MONGODB_NOSQL_SECURE_MARK = getNextSecureMark()
 
 function iterateObjectStrings (target, fn, levelKeys = [], depth = 50, visited = new Set()) {
@@ -37,34 +37,39 @@ class NosqlInjectionMongodbAnalyzer extends InjectionAnalyzer {
   onConfigure () {
     this.configureSanitizers()
 
-    this.addSub('datadog:mongodb:collection:filter:start', ({ filters }) => {
+    const onStart = ({ filters }) => {
       const store = storage.getStore()
       if (store && !store.nosqlAnalyzed && filters?.length) {
         filters.forEach(filter => {
           this.analyze({ filter }, store)
         })
       }
-    })
 
-    this.addSub('datadog:mongoose:model:filter:start', ({ filters }) => {
-      const store = storage.getStore()
-      if (!store) return
+      return store
+    }
 
-      if (filters?.length) {
-        filters.forEach(filter => {
-          this.analyze({ filter }, store)
-        })
+    const onStartAndEnterWithStore = (message) => {
+      const store = onStart(message || {})
+      if (store) {
+        storage.enterWith({ ...store, nosqlAnalyzed: true, nosqlParentStore: store })
       }
+    }
 
-      storage.enterWith({ ...store, nosqlAnalyzed: true, mongooseParentStore: store })
-    })
-
-    this.addSub('datadog:mongoose:model:filter:finish', () => {
+    const onFinish = () => {
       const store = storage.getStore()
-      if (store?.mongooseParentStore) {
-        storage.enterWith(store.mongooseParentStore)
+      if (store?.nosqlParentStore) {
+        storage.enterWith(store.nosqlParentStore)
       }
-    })
+    }
+
+    this.addSub('datadog:mongodb:collection:filter:start', onStart)
+
+    this.addSub('datadog:mongoose:model:filter:start', onStartAndEnterWithStore)
+    this.addSub('datadog:mongoose:model:filter:finish', onFinish)
+
+    this.addSub('datadog:mquery:filter:prepare', onStart)
+    this.addSub('tracing:datadog:mquery:filter:start', onStartAndEnterWithStore)
+    this.addSub('tracing:datadog:mquery:filter:asyncEnd', onFinish)
   }
 
   configureSanitizers () {

package/packages/dd-trace/src/datastreams/processor.js

@@ -125,6 +125,21 @@ function getSizeOrZero (obj) {
   if (Buffer.isBuffer(obj)) {
     return obj.length
   }
+  if (Array.isArray(obj) && obj.length > 0) {
+    if (typeof obj[0] === 'number') return Buffer.from(obj).length
+    let payloadSize = 0
+    obj.forEach(item => {
+      payloadSize += getSizeOrZero(item)
+    })
+    return payloadSize
+  }
+  if (typeof obj === 'object') {
+    try {
+      return getHeadersSize(obj)
+    } catch {
+      // pass
+    }
+  }
   return 0
 }
 
@@ -157,7 +172,8 @@ class DataStreamsProcessor {
     env,
     tags,
     version,
-    service
+    service,
+    flushInterval
   } = {}) {
     this.writer = new DataStreamsWriter({
       hostname,
@@ -173,11 +189,13 @@ class DataStreamsProcessor {
     this.service = service || 'unnamed-nodejs-service'
     this.version = version || ''
     this.sequence = 0
+    this.flushInterval = flushInterval
 
     if (this.enabled) {
-      this.timer = setInterval(this.onInterval.bind(this), 10000)
+      this.timer = setInterval(this.onInterval.bind(this), flushInterval)
       this.timer.unref()
     }
+    process.once('beforeExit', () => this.onInterval())
   }
 
   onInterval () {
@@ -201,7 +219,8 @@ class DataStreamsProcessor {
    */
   bucketFromTimestamp (timestamp) {
     const bucketTime = Math.round(timestamp - (timestamp % this.bucketSizeNs))
-    return this.buckets.forTime(bucketTime)
+    const bucket = this.buckets.forTime(bucketTime)
+    return bucket
   }
 
   recordCheckpoint (checkpoint, span = null) {
@@ -259,8 +278,10 @@ class DataStreamsProcessor {
     }
     if (direction === 'direction:out') {
       // Add the header for this now, as the callee doesn't have access to context when producing
-      payloadSize += getSizeOrZero(encodePathwayContext(dataStreamsContext))
-      payloadSize += CONTEXT_PROPAGATION_KEY.length
+      // - 1 to account for extra byte for {
+      const ddInfoContinued = {}
+      ddInfoContinued[CONTEXT_PROPAGATION_KEY] = encodePathwayContext(dataStreamsContext).toJSON()
+      payloadSize += getSizeOrZero(JSON.stringify(ddInfoContinued)) - 1
     }
     const checkpoint = {
       currentTimestamp: nowNs,
@@ -322,6 +343,10 @@ class DataStreamsProcessor {
       Stats: serializedBuckets
     }
   }
+
+  setUrl (url) {
+    this.writer.setUrl(url)
+  }
 }
 
 module.exports = {

package/packages/dd-trace/src/datastreams/writer.js

@@ -59,6 +59,15 @@ class DataStreamsWriter {
       })
     })
   }
+
+  setUrl (url) {
+    try {
+      url = new URL(url)
+      this._url = url
+    } catch (e) {
+      log.warn(e.stack)
+    }
+  }
 }
 
 module.exports = {

package/packages/dd-trace/src/opentracing/propagation/text_map.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const pick = require('lodash.pick')
+const pick = require('../../../../utils/src/pick')
 const id = require('../../id')
 const DatadogSpanContext = require('../span_context')
 const log = require('../../log')

package/packages/dd-trace/src/plugins/util/web.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const uniq = require('lodash.uniq')
+const uniq = require('../../../../utils/src/uniq')
 const analyticsSampler = require('../../analytics_sampler')
 const FORMAT_HTTP_HEADERS = 'http_headers'
 const log = require('../../log')

package/packages/dd-trace/src/tracer.js

@@ -135,6 +135,7 @@ class DatadogTracer extends Tracer {
 
   setUrl (url) {
     this._exporter.setUrl(url)
+    this._dataStreamsProcessor.setUrl(url)
   }
 
   scope () {

package/packages/utils/src/kebabcase.js

@@ -0,0 +1,16 @@
+'use strict'
+
+module.exports = str => {
+  if (typeof str !== 'string') {
+    throw new TypeError('Expected a string')
+  }
+
+  return str
+    .trim()
+    .replace(/([a-z])([A-Z])/g, '$1-$2')
+    .replace(/\s+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .replace(/_/g, '-')
+    .replace(/-{2,}/g, '-')
+    .toLowerCase()
+}

package/packages/utils/src/pick.js

@@ -0,0 +1,11 @@
+'use strict'
+
+module.exports = (object, props) => {
+  const result = {}
+  props.forEach(prop => {
+    if (prop in object) {
+      result[prop] = object[prop]
+    }
+  })
+  return result
+}

package/packages/utils/src/uniq.js

@@ -0,0 +1,5 @@
+'use strict'
+
+module.exports = function (arr) {
+  return [...new Set(arr)]
+}