dd-trace 4.16.0 → 4.17.0

package/LICENSE-3rdparty.csv

@@ -14,6 +14,7 @@ require,import-in-the-middle,Apache license 2.0,Copyright 2021 Datadog Inc.
 require,int64-buffer,MIT,Copyright 2015-2016 Yusuke Kawasaki
 require,ipaddr.js,MIT,Copyright 2011-2017 whitequark
 require,istanbul-lib-coverage,BSD-3-Clause,Copyright 2012-2015 Yahoo! Inc.
+require,jest-docblock,MIT,Copyright Meta Platforms, Inc. and affiliates.
 require,koalas,MIT,Copyright 2013-2017 Brian Woodward
 require,limiter,MIT,Copyright 2011 John Hurliman
 require,lodash.kebabcase,MIT,Copyright JS Foundation and other contributors

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "4.16.0",
+  "version": "4.17.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -70,9 +70,9 @@
   "dependencies": {
     "@datadog/native-appsec": "^4.0.0",
     "@datadog/native-iast-rewriter": "2.1.3",
-    "@datadog/native-iast-taint-tracking": "1.5.0",
+    "@datadog/native-iast-taint-tracking": "1.6.1",
     "@datadog/native-metrics": "^2.0.0",
-    "@datadog/pprof": "3.2.0",
+    "@datadog/pprof": "4.0.0",
     "@datadog/sketches-js": "^2.1.0",
     "@opentelemetry/api": "^1.0.0",
     "@opentelemetry/core": "^1.14.0",
@@ -83,6 +83,7 @@
     "int64-buffer": "^0.1.9",
     "ipaddr.js": "^2.1.0",
     "istanbul-lib-coverage": "3.2.0",
+    "jest-docblock": "^29.7.0",
     "koalas": "^1.0.2",
     "limiter": "^1.1.4",
     "lodash.kebabcase": "^4.1.1",

package/packages/datadog-instrumentations/src/body-parser.js

@@ -10,8 +10,9 @@ function publishRequestBodyAndNext (req, res, next) {
   return function () {
     if (bodyParserReadCh.hasSubscribers && req) {
       const abortController = new AbortController()
+      const body = req.body
 
-      bodyParserReadCh.publish({ req, res, abortController })
+      bodyParserReadCh.publish({ req, res, body, abortController })
 
       if (abortController.signal.aborted) return
     }

package/packages/datadog-instrumentations/src/cucumber.js

@@ -31,6 +31,10 @@ const {
   getTestSuitePath
 } = require('../../dd-trace/src/plugins/util/test')
 
+const isMarkedAsUnskippable = (pickle) => {
+  return !!pickle.tags.find(tag => tag.name === '@datadog:unskippable')
+}
+
 // We'll preserve the original coverage here
 const originalCoverageMap = createCoverageMap()
 
@@ -39,6 +43,9 @@ const patched = new WeakSet()
 
 let pickleByFile = {}
 const pickleResultByFile = {}
+let skippableSuites = []
+let isForcedToRun = false
+let isUnskippable = false
 
 function getSuiteStatusFromTestStatuses (testStatuses) {
   if (testStatuses.some(status => status === 'fail')) {
@@ -91,7 +98,11 @@ function wrapRun (pl, isLatestVersion) {
       const testSuiteFullPath = this.pickle.uri
 
       if (!pickleResultByFile[testSuiteFullPath]) { // first test in suite
-        testSuiteStartCh.publish(testSuiteFullPath)
+        isUnskippable = isMarkedAsUnskippable(this.pickle)
+        const testSuitePath = getTestSuitePath(testSuiteFullPath, process.cwd())
+        isForcedToRun = isUnskippable && skippableSuites.includes(testSuitePath)
+
+        testSuiteStartCh.publish({ testSuitePath, isUnskippable, isForcedToRun })
       }
 
       const testSourceLine = this.gherkinDocument &&
@@ -221,8 +232,11 @@ function getFilteredPickles (runtime, suitesToSkip) {
   return runtime.pickleIds.reduce((acc, pickleId) => {
     const test = runtime.eventDataCollector.getPickle(pickleId)
     const testSuitePath = getTestSuitePath(test.uri, process.cwd())
+
+    const isUnskippable = isMarkedAsUnskippable(test)
     const isSkipped = suitesToSkip.includes(testSuitePath)
-    if (isSkipped) {
+
+    if (isSkipped && !isUnskippable) {
       acc.skippedSuites.add(testSuitePath)
     } else {
       acc.picklesToRun.push(pickleId)
@@ -270,7 +284,11 @@ addHook({
       skippableSuitesCh.publish({ onDone })
     })
 
-    const { err, skippableSuites } = await skippableSuitesPromise
+    const skippableResponse = await skippableSuitesPromise
+
+    const err = skippableResponse.err
+    skippableSuites = skippableResponse.skippableSuites
+
     let skippedSuites = []
     let isSuitesSkipped = false
 
@@ -315,7 +333,9 @@ addHook({
         status: success ? 'pass' : 'fail',
         isSuitesSkipped,
         testCodeCoverageLinesTotal,
-        numSkippedSuites: skippedSuites.length
+        numSkippedSuites: skippedSuites.length,
+        hasUnskippableSuites: isUnskippable,
+        hasForcedToRunSuites: isForcedToRun
       })
     })
     return success

package/packages/datadog-instrumentations/src/express-mongo-sanitize.js

@@ -0,0 +1,45 @@
+'use strict'
+
+const {
+  channel,
+  addHook
+} = require('./helpers/instrument')
+const shimmer = require('../../datadog-shimmer')
+
+const sanitizeMethodFinished = channel('datadog:express-mongo-sanitize:sanitize:finish')
+const sanitizeMiddlewareFinished = channel('datadog:express-mongo-sanitize:filter:finish')
+
+const propertiesToSanitize = ['body', 'params', 'headers', 'query']
+
+addHook({ name: 'express-mongo-sanitize', versions: ['>=1.0.0'] }, expressMongoSanitize => {
+  shimmer.wrap(expressMongoSanitize, 'sanitize', sanitize => function () {
+    const sanitizedObject = sanitize.apply(this, arguments)
+
+    if (sanitizeMethodFinished.hasSubscribers) {
+      sanitizeMethodFinished.publish({ sanitizedObject })
+    }
+
+    return sanitizedObject
+  })
+
+  return shimmer.wrap(expressMongoSanitize, function () {
+    const middleware = expressMongoSanitize.apply(this, arguments)
+
+    return shimmer.wrap(middleware, function (req, res, next) {
+      if (!sanitizeMiddlewareFinished.hasSubscribers) {
+        return middleware.apply(this, arguments)
+      }
+
+      const wrappedNext = shimmer.wrap(next, function () {
+        sanitizeMiddlewareFinished.publish({
+          sanitizedProperties: propertiesToSanitize,
+          req
+        })
+
+        return next.apply(this, arguments)
+      })
+
+      return middleware.call(this, req, res, wrappedNext)
+    })
+  })
+})

package/packages/datadog-instrumentations/src/express.js

@@ -33,8 +33,9 @@ function publishQueryParsedAndNext (req, res, next) {
   return function () {
     if (queryParserReadCh.hasSubscribers && req) {
       const abortController = new AbortController()
+      const query = req.query
 
-      queryParserReadCh.publish({ req, res, abortController })
+      queryParserReadCh.publish({ req, res, query, abortController })
 
       if (abortController.signal.aborted) return
     }

package/packages/datadog-instrumentations/src/helpers/hooks.js

@@ -38,6 +38,7 @@ module.exports = {
   'dns': () => require('../dns'),
   'elasticsearch': () => require('../elasticsearch'),
   'express': () => require('../express'),
+  'express-mongo-sanitize': () => require('../express-mongo-sanitize'),
   'fastify': () => require('../fastify'),
   'find-my-way': () => require('../find-my-way'),
   'fs': () => require('../fs'),
@@ -68,7 +69,7 @@ module.exports = {
   'mocha': () => require('../mocha'),
   'mocha-each': () => require('../mocha'),
   'moleculer': () => require('../moleculer'),
-  'mongodb': () => require('../mongodb-core'),
+  'mongodb': () => require('../mongodb'),
   'mongodb-core': () => require('../mongodb-core'),
   'mongoose': () => require('../mongoose'),
   'mysql': () => require('../mysql'),

package/packages/datadog-instrumentations/src/jest.js

@@ -46,6 +46,8 @@ let isCodeCoverageEnabled = false
 let isSuitesSkippingEnabled = false
 let isSuitesSkipped = false
 let numSkippedSuites = 0
+let hasUnskippableSuites = false
+let hasForcedToRunSuites = false
 
 const sessionAsyncResource = new AsyncResource('bound-anonymous-fn')
 
@@ -205,15 +207,17 @@ addHook({
     const [test] = shardedTests
     const rootDir = test && test.context && test.context.config && test.context.config.rootDir
 
-    const { skippedSuites, suitesToRun } = getJestSuitesToRun(skippableSuites, shardedTests, rootDir || process.cwd())
+    const jestSuitesToRun = getJestSuitesToRun(skippableSuites, shardedTests, rootDir || process.cwd())
+    hasUnskippableSuites = jestSuitesToRun.hasUnskippableSuites
+    hasForcedToRunSuites = jestSuitesToRun.hasForcedToRunSuites
 
-    isSuitesSkipped = suitesToRun.length !== shardedTests.length
-    numSkippedSuites = skippedSuites.length
+    isSuitesSkipped = jestSuitesToRun.suitesToRun.length !== shardedTests.length
+    numSkippedSuites = jestSuitesToRun.skippedSuites.length
 
-    itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion })
+    itrSkippedSuitesCh.publish({ skippedSuites: jestSuitesToRun.skippedSuites, frameworkVersion })
 
     skippableSuites = []
-    return suitesToRun
+    return jestSuitesToRun.suitesToRun
   })
   return sequencerPackage
 })
@@ -285,7 +289,9 @@ function cliWrapper (cli, jestVersion) {
         isSuitesSkippingEnabled,
         isCodeCoverageEnabled,
         testCodeCoverageLinesTotal,
-        numSkippedSuites
+        numSkippedSuites,
+        hasUnskippableSuites,
+        hasForcedToRunSuites
       })
     })
 
@@ -500,16 +506,19 @@ addHook({
     const testPaths = await getTestPaths.apply(this, arguments)
     const { tests } = testPaths
 
-    const { skippedSuites, suitesToRun } = getJestSuitesToRun(skippableSuites, tests, rootDir)
+    const jestSuitesToRun = getJestSuitesToRun(skippableSuites, tests, rootDir)
 
-    isSuitesSkipped = suitesToRun.length !== tests.length
-    numSkippedSuites = skippedSuites.length
+    hasUnskippableSuites = jestSuitesToRun.hasUnskippableSuites
+    hasForcedToRunSuites = jestSuitesToRun.hasForcedToRunSuites
 
-    itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion })
+    isSuitesSkipped = jestSuitesToRun.suitesToRun.length !== tests.length
+    numSkippedSuites = jestSuitesToRun.skippedSuites.length
+
+    itrSkippedSuitesCh.publish({ skippedSuites: jestSuitesToRun.skippedSuites, frameworkVersion })
 
     skippableSuites = []
 
-    return { ...testPaths, tests: suitesToRun }
+    return { ...testPaths, tests: jestSuitesToRun.suitesToRun }
   })
 
   return searchSourcePackage

package/packages/datadog-instrumentations/src/knex.js

@@ -1,9 +1,12 @@
 'use strict'
 
-const { addHook } = require('./helpers/instrument')
+const { addHook, channel } = require('./helpers/instrument')
 const { wrapThen } = require('./helpers/promise')
 const shimmer = require('../../datadog-shimmer')
 
+const startRawQueryCh = channel('datadog:knex:raw:start')
+const finishRawQueryCh = channel('datadog:knex:raw:finish')
+
 patch('lib/query/builder.js')
 patch('lib/raw.js')
 patch('lib/schema/builder.js')
@@ -18,3 +21,61 @@ function patch (file) {
     return Builder
   })
 }
+
+addHook({
+  name: 'knex',
+  versions: ['>=2'],
+  file: 'lib/knex-builder/Knex.js'
+}, Knex => {
+  shimmer.wrap(Knex.Client.prototype, 'raw', raw => function () {
+    if (!startRawQueryCh.hasSubscribers) {
+      return raw.apply(this, arguments)
+    }
+
+    const sql = arguments[0]
+
+    // Skip query done by Knex to get the value used for undefined
+    if (sql === 'DEFAULT') {
+      return raw.apply(this, arguments)
+    }
+
+    function finish () {
+      finishRawQueryCh.publish()
+    }
+
+    startRawQueryCh.publish({ sql, dialect: this.dialect })
+
+    const rawResult = raw.apply(this, arguments)
+
+    shimmer.wrap(rawResult, 'then', originalThen => function () {
+      arguments[0] = wrapCallbackWithFinish(arguments[0], finish)
+      arguments[1] = wrapCallbackWithFinish(arguments[1], finish)
+
+      const originalThenResult = originalThen.apply(this, arguments)
+
+      shimmer.wrap(originalThenResult, 'catch', originalCatch => function () {
+        arguments[0] = wrapCallbackWithFinish(arguments[0], finish)
+        return originalCatch.apply(this, arguments)
+      })
+
+      return originalThenResult
+    })
+
+    shimmer.wrap(rawResult, 'asCallback', originalAsCallback => function () {
+      arguments[0] = wrapCallbackWithFinish(arguments[0], finish)
+      return originalAsCallback.apply(this, arguments)
+    })
+
+    return rawResult
+  })
+  return Knex
+})
+
+function wrapCallbackWithFinish (callback, finish) {
+  if (typeof callback !== 'function') return callback
+
+  return function () {
+    finish()
+    callback.apply(this, arguments)
+  }
+}

package/packages/datadog-instrumentations/src/mocha.js

@@ -1,9 +1,10 @@
 const { createCoverageMap } = require('istanbul-lib-coverage')
 
+const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util')
+
 const { addHook, channel, AsyncResource } = require('./helpers/instrument')
 const shimmer = require('../../datadog-shimmer')
 const log = require('../../dd-trace/src/log')
-
 const {
   getCoveredFilenamesFromCoverage,
   resetCoverage,
@@ -50,6 +51,8 @@ let suitesToSkip = []
 let frameworkVersion
 let isSuitesSkipped = false
 let skippedSuites = []
+const unskippableSuites = []
+let isForcedToRun = false
 
 function getSuitesByTestFile (root) {
   const suitesByTestFile = {}
@@ -104,7 +107,8 @@ function getFilteredSuites (originalSuites) {
   return originalSuites.reduce((acc, suite) => {
     const testPath = getTestSuitePath(suite.file, process.cwd())
     const shouldSkip = suitesToSkip.includes(testPath)
-    if (shouldSkip) {
+    const isUnskippable = unskippableSuites.includes(suite.file)
+    if (shouldSkip && !isUnskippable) {
       acc.skippedSuites.add(testPath)
     } else {
       acc.suitesToRun.push(suite)
@@ -151,7 +155,9 @@ function mochaHook (Runner) {
         status,
         isSuitesSkipped,
         testCodeCoverageLinesTotal,
-        numSkippedSuites: skippedSuites.length
+        numSkippedSuites: skippedSuites.length,
+        hasForcedToRunSuites: isForcedToRun,
+        hasUnskippableSuites: !!unskippableSuites.length
       })
     }))
 
@@ -172,8 +178,10 @@ function mochaHook (Runner) {
       if (!asyncResource) {
         asyncResource = new AsyncResource('bound-anonymous-fn')
         testFileToSuiteAr.set(suite.file, asyncResource)
+        const isUnskippable = unskippableSuites.includes(suite.file)
+        isForcedToRun = isUnskippable && suitesToSkip.includes(getTestSuitePath(suite.file, process.cwd()))
         asyncResource.runInAsyncScope(() => {
-          testSuiteStartCh.publish(suite)
+          testSuiteStartCh.publish({ testSuite: suite.file, isUnskippable, isForcedToRun })
         })
       }
     })
@@ -370,6 +378,13 @@ addHook({
 
     const runner = run.apply(this, arguments)
 
+    this.files.forEach(path => {
+      const isUnskippable = isMarkedAsUnskippable({ path })
+      if (isUnskippable) {
+        unskippableSuites.push(path)
+      }
+    })
+
     const onReceivedSkippableSuites = ({ err, skippableSuites }) => {
       if (err) {
         suitesToSkip = []

package/packages/datadog-instrumentations/src/mongodb.js

@@ -0,0 +1,63 @@
+'use strict'
+
+require('./mongodb-core')
+
+const {
+  channel,
+  addHook,
+  AsyncResource
+} = require('./helpers/instrument')
+const shimmer = require('../../datadog-shimmer')
+
+// collection methods with filter
+const collectionMethodsWithFilter = [
+  'count',
+  'countDocuments',
+  'deleteMany',
+  'deleteOne',
+  'find',
+  'findOneAndDelete',
+  'findOneAndReplace',
+  'replaceOne'
+] // findOne is ignored because it calls to find
+
+const collectionMethodsWithTwoFilters = [
+  'findOneAndUpdate',
+  'updateMany',
+  'updateOne'
+]
+
+const startCh = channel('datadog:mongodb:collection:filter:start')
+
+addHook({ name: 'mongodb', versions: ['>=3.3 <5', '5', '>=6'] }, mongodb => {
+  [...collectionMethodsWithFilter, ...collectionMethodsWithTwoFilters].forEach(methodName => {
+    if (!(methodName in mongodb.Collection.prototype)) return
+
+    const useTwoArguments = collectionMethodsWithTwoFilters.includes(methodName)
+
+    shimmer.wrap(mongodb.Collection.prototype, methodName, method => {
+      return function () {
+        if (!startCh.hasSubscribers) {
+          return method.apply(this, arguments)
+        }
+
+        const asyncResource = new AsyncResource('bound-anonymous-fn')
+
+        return asyncResource.runInAsyncScope(() => {
+          const filters = [arguments[0]]
+          if (useTwoArguments) {
+            filters.push(arguments[1])
+          }
+
+          startCh.publish({
+            filters,
+            methodName
+          })
+
+          return method.apply(this, arguments)
+        })
+      }
+    })
+  })
+  return mongodb
+})

package/packages/datadog-instrumentations/src/mongoose.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { addHook } = require('./helpers/instrument')
+const { addHook, channel } = require('./helpers/instrument')
 const { wrapThen } = require('./helpers/promise')
 const { AsyncResource } = require('./helpers/instrument')
 const shimmer = require('../../datadog-shimmer')
@@ -26,5 +26,144 @@ addHook({
   }
 
   shimmer.wrap(mongoose.Collection.prototype, 'addQueue', wrapAddQueue)
+
   return mongoose
 })
+
+const startCh = channel('datadog:mongoose:model:filter:start')
+const finishCh = channel('datadog:mongoose:model:filter:finish')
+
+const collectionMethodsWithFilter = [
+  'count',
+  'countDocuments',
+  'deleteMany',
+  'deleteOne',
+  'find',
+  'findOne',
+  'findOneAndDelete',
+  'findOneAndReplace',
+  'replaceOne',
+  'remove'
+]
+
+const collectionMethodsWithTwoFilters = [
+  'findOneAndUpdate',
+  'updateMany',
+  'updateOne'
+]
+
+addHook({
+  name: 'mongoose',
+  versions: ['>=4.6.4 <5', '5', '6', '>=7'],
+  file: 'lib/model.js'
+}, Model => {
+  [...collectionMethodsWithFilter, ...collectionMethodsWithTwoFilters].forEach(methodName => {
+    const useTwoArguments = collectionMethodsWithTwoFilters.includes(methodName)
+    if (!(methodName in Model)) return
+
+    shimmer.wrap(Model, methodName, method => {
+      return function wrappedModelMethod () {
+        if (!startCh.hasSubscribers) {
+          return method.apply(this, arguments)
+        }
+
+        const asyncResource = new AsyncResource('bound-anonymous-fn')
+
+        const filters = [arguments[0]]
+        if (useTwoArguments) {
+          filters.push(arguments[1])
+        }
+
+        const finish = asyncResource.bind(function () {
+          finishCh.publish()
+        })
+
+        let callbackWrapped = false
+        const lastArgumentIndex = arguments.length - 1
+
+        if (typeof arguments[lastArgumentIndex] === 'function') {
+          // is a callback, wrap it to execute finish()
+          shimmer.wrap(arguments, lastArgumentIndex, originalCb => {
+            return function () {
+              finish()
+
+              return originalCb.apply(this, arguments)
+            }
+          })
+
+          callbackWrapped = true
+        }
+
+        return asyncResource.runInAsyncScope(() => {
+          startCh.publish({
+            filters,
+            methodName
+          })
+
+          const res = method.apply(this, arguments)
+
+          // if it is not callback, wrap exec method and its then
+          if (!callbackWrapped) {
+            shimmer.wrap(res, 'exec', originalExec => {
+              return function wrappedExec () {
+                const execResult = originalExec.apply(this, arguments)
+
+                // wrap them method, wrap resolve and reject methods
+                shimmer.wrap(execResult, 'then', originalThen => {
+                  return function wrappedThen () {
+                    const resolve = arguments[0]
+                    const reject = arguments[1]
+
+                    // not using shimmer here because resolve/reject could be empty
+                    arguments[0] = function wrappedResolve () {
+                      finish()
+
+                      if (resolve) {
+                        return resolve.apply(this, arguments)
+                      }
+                    }
+
+                    arguments[1] = function wrappedReject () {
+                      finish()
+
+                      if (reject) {
+                        return reject.apply(this, arguments)
+                      }
+                    }
+
+                    return originalThen.apply(this, arguments)
+                  }
+                })
+
+                return execResult
+              }
+            })
+          }
+          return res
+        })
+      }
+    })
+  })
+
+  return Model
+})
+
+const sanitizeFilterFinishCh = channel('datadog:mongoose:sanitize-filter:finish')
+
+addHook({
+  name: 'mongoose',
+  versions: ['6', '>=7'],
+  file: 'lib/helpers/query/sanitizeFilter.js'
+}, sanitizeFilter => {
+  return shimmer.wrap(sanitizeFilter, function wrappedSanitizeFilter () {
+    const sanitizedObject = sanitizeFilter.apply(this, arguments)
+
+    if (sanitizeFilterFinishCh.hasSubscribers) {
+      sanitizeFilterFinishCh.publish({
+        sanitizedObject
+      })
+    }
+
+    return sanitizedObject
+  })
+})

package/packages/datadog-instrumentations/src/next.js

@@ -10,6 +10,8 @@ const startChannel = channel('apm:next:request:start')
 const finishChannel = channel('apm:next:request:finish')
 const errorChannel = channel('apm:next:request:error')
 const pageLoadChannel = channel('apm:next:page:load')
+const bodyParsedChannel = channel('apm:next:body-parsed')
+const queryParsedChannel = channel('apm:next:query-parsed')
 
 const requests = new WeakSet()
 
@@ -212,3 +214,41 @@ addHook({
 
   return nextServer
 })
+
+addHook({
+  name: 'next',
+  versions: ['>=13'],
+  file: 'dist/server/web/spec-extension/request.js'
+}, request => {
+  const nextUrlDescriptor = Object.getOwnPropertyDescriptor(request.NextRequest.prototype, 'nextUrl')
+  shimmer.wrap(nextUrlDescriptor, 'get', function (originalGet) {
+    return function wrappedGet () {
+      const nextUrl = originalGet.apply(this, arguments)
+      if (queryParsedChannel.hasSubscribers) {
+        const query = {}
+        for (const key of nextUrl.searchParams.keys()) {
+          if (!query[key]) {
+            query[key] = nextUrl.searchParams.getAll(key)
+          }
+        }
+
+        queryParsedChannel.publish({ query })
+      }
+      return nextUrl
+    }
+  })
+
+  Object.defineProperty(request.NextRequest.prototype, 'nextUrl', nextUrlDescriptor)
+
+  shimmer.massWrap(request.NextRequest.prototype, ['text', 'json'], function (originalMethod) {
+    return async function wrappedJson () {
+      const body = await originalMethod.apply(this, arguments)
+      bodyParsedChannel.publish({
+        body
+      })
+      return body
+    }
+  })
+
+  return request
+})