dd-trace 4.16.0 → 4.18.0

package/LICENSE-3rdparty.csv

@@ -14,6 +14,7 @@ require,import-in-the-middle,Apache license 2.0,Copyright 2021 Datadog Inc.
 require,int64-buffer,MIT,Copyright 2015-2016 Yusuke Kawasaki
 require,ipaddr.js,MIT,Copyright 2011-2017 whitequark
 require,istanbul-lib-coverage,BSD-3-Clause,Copyright 2012-2015 Yahoo! Inc.
+require,jest-docblock,MIT,Copyright Meta Platforms, Inc. and affiliates.
 require,koalas,MIT,Copyright 2013-2017 Brian Woodward
 require,limiter,MIT,Copyright 2011 John Hurliman
 require,lodash.kebabcase,MIT,Copyright JS Foundation and other contributors

package/index.d.ts

@@ -453,7 +453,15 @@ export declare interface TracerOptions {
        * Whether to enable vulnerability redaction
        * @default true
        */
-      redactionEnabled?: boolean
+      redactionEnabled?: boolean,
+      /**
+       * Specifies a regex that will redact sensitive source names in vulnerability reports.
+       */
+      redactionNamePattern?: string,
+      /**
+       * Specifies a regex that will redact sensitive source values in vulnerability reports.
+       */
+      redactionValuePattern?: string
     }
   };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "4.16.0",
+  "version": "4.18.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -69,10 +69,10 @@
   },
   "dependencies": {
     "@datadog/native-appsec": "^4.0.0",
-    "@datadog/native-iast-rewriter": "2.1.3",
-    "@datadog/native-iast-taint-tracking": "1.5.0",
+    "@datadog/native-iast-rewriter": "2.2.1",
+    "@datadog/native-iast-taint-tracking": "1.6.3",
     "@datadog/native-metrics": "^2.0.0",
-    "@datadog/pprof": "3.2.0",
+    "@datadog/pprof": "4.0.1",
     "@datadog/sketches-js": "^2.1.0",
     "@opentelemetry/api": "^1.0.0",
     "@opentelemetry/core": "^1.14.0",
@@ -83,6 +83,7 @@
     "int64-buffer": "^0.1.9",
     "ipaddr.js": "^2.1.0",
     "istanbul-lib-coverage": "3.2.0",
+    "jest-docblock": "^29.7.0",
     "koalas": "^1.0.2",
     "limiter": "^1.1.4",
     "lodash.kebabcase": "^4.1.1",

package/packages/datadog-instrumentations/src/body-parser.js

@@ -10,8 +10,9 @@ function publishRequestBodyAndNext (req, res, next) {
   return function () {
     if (bodyParserReadCh.hasSubscribers && req) {
       const abortController = new AbortController()
+      const body = req.body
 
-      bodyParserReadCh.publish({ req, res, abortController })
+      bodyParserReadCh.publish({ req, res, body, abortController })
 
       if (abortController.signal.aborted) return
     }

package/packages/datadog-instrumentations/src/cucumber.js

@@ -31,6 +31,10 @@ const {
   getTestSuitePath
 } = require('../../dd-trace/src/plugins/util/test')
 
+const isMarkedAsUnskippable = (pickle) => {
+  return !!pickle.tags.find(tag => tag.name === '@datadog:unskippable')
+}
+
 // We'll preserve the original coverage here
 const originalCoverageMap = createCoverageMap()
 
@@ -39,6 +43,9 @@ const patched = new WeakSet()
 
 let pickleByFile = {}
 const pickleResultByFile = {}
+let skippableSuites = []
+let isForcedToRun = false
+let isUnskippable = false
 
 function getSuiteStatusFromTestStatuses (testStatuses) {
   if (testStatuses.some(status => status === 'fail')) {
@@ -91,7 +98,11 @@ function wrapRun (pl, isLatestVersion) {
       const testSuiteFullPath = this.pickle.uri
 
       if (!pickleResultByFile[testSuiteFullPath]) { // first test in suite
-        testSuiteStartCh.publish(testSuiteFullPath)
+        isUnskippable = isMarkedAsUnskippable(this.pickle)
+        const testSuitePath = getTestSuitePath(testSuiteFullPath, process.cwd())
+        isForcedToRun = isUnskippable && skippableSuites.includes(testSuitePath)
+
+        testSuiteStartCh.publish({ testSuitePath, isUnskippable, isForcedToRun })
       }
 
       const testSourceLine = this.gherkinDocument &&
@@ -221,8 +232,11 @@ function getFilteredPickles (runtime, suitesToSkip) {
   return runtime.pickleIds.reduce((acc, pickleId) => {
     const test = runtime.eventDataCollector.getPickle(pickleId)
     const testSuitePath = getTestSuitePath(test.uri, process.cwd())
+
+    const isUnskippable = isMarkedAsUnskippable(test)
     const isSkipped = suitesToSkip.includes(testSuitePath)
-    if (isSkipped) {
+
+    if (isSkipped && !isUnskippable) {
       acc.skippedSuites.add(testSuitePath)
     } else {
       acc.picklesToRun.push(pickleId)
@@ -270,7 +284,11 @@ addHook({
       skippableSuitesCh.publish({ onDone })
     })
 
-    const { err, skippableSuites } = await skippableSuitesPromise
+    const skippableResponse = await skippableSuitesPromise
+
+    const err = skippableResponse.err
+    skippableSuites = skippableResponse.skippableSuites
+
     let skippedSuites = []
     let isSuitesSkipped = false
 
@@ -278,6 +296,11 @@ addHook({
       const filteredPickles = getFilteredPickles(this, skippableSuites)
       const { picklesToRun } = filteredPickles
       isSuitesSkipped = picklesToRun.length !== this.pickleIds.length
+
+      log.debug(
+        () => `${picklesToRun.length} out of ${this.pickleIds.length} suites are going to run.`
+      )
+
       this.pickleIds = picklesToRun
 
       skippedSuites = Array.from(filteredPickles.skippedSuites)
@@ -315,7 +338,9 @@ addHook({
         status: success ? 'pass' : 'fail',
         isSuitesSkipped,
         testCodeCoverageLinesTotal,
-        numSkippedSuites: skippedSuites.length
+        numSkippedSuites: skippedSuites.length,
+        hasUnskippableSuites: isUnskippable,
+        hasForcedToRunSuites: isForcedToRun
       })
     })
     return success

package/packages/datadog-instrumentations/src/express-mongo-sanitize.js

@@ -0,0 +1,45 @@
+'use strict'
+
+const {
+  channel,
+  addHook
+} = require('./helpers/instrument')
+const shimmer = require('../../datadog-shimmer')
+
+const sanitizeMethodFinished = channel('datadog:express-mongo-sanitize:sanitize:finish')
+const sanitizeMiddlewareFinished = channel('datadog:express-mongo-sanitize:filter:finish')
+
+const propertiesToSanitize = ['body', 'params', 'headers', 'query']
+
+addHook({ name: 'express-mongo-sanitize', versions: ['>=1.0.0'] }, expressMongoSanitize => {
+  shimmer.wrap(expressMongoSanitize, 'sanitize', sanitize => function () {
+    const sanitizedObject = sanitize.apply(this, arguments)
+
+    if (sanitizeMethodFinished.hasSubscribers) {
+      sanitizeMethodFinished.publish({ sanitizedObject })
+    }
+
+    return sanitizedObject
+  })
+
+  return shimmer.wrap(expressMongoSanitize, function () {
+    const middleware = expressMongoSanitize.apply(this, arguments)
+
+    return shimmer.wrap(middleware, function (req, res, next) {
+      if (!sanitizeMiddlewareFinished.hasSubscribers) {
+        return middleware.apply(this, arguments)
+      }
+
+      const wrappedNext = shimmer.wrap(next, function () {
+        sanitizeMiddlewareFinished.publish({
+          sanitizedProperties: propertiesToSanitize,
+          req
+        })
+
+        return next.apply(this, arguments)
+      })
+
+      return middleware.call(this, req, res, wrappedNext)
+    })
+  })
+})

package/packages/datadog-instrumentations/src/express.js

@@ -33,8 +33,9 @@ function publishQueryParsedAndNext (req, res, next) {
   return function () {
     if (queryParserReadCh.hasSubscribers && req) {
       const abortController = new AbortController()
+      const query = req.query
 
-      queryParserReadCh.publish({ req, res, abortController })
+      queryParserReadCh.publish({ req, res, query, abortController })
 
       if (abortController.signal.aborted) return
     }

package/packages/datadog-instrumentations/src/helpers/hooks.js

@@ -38,6 +38,7 @@ module.exports = {
   'dns': () => require('../dns'),
   'elasticsearch': () => require('../elasticsearch'),
   'express': () => require('../express'),
+  'express-mongo-sanitize': () => require('../express-mongo-sanitize'),
   'fastify': () => require('../fastify'),
   'find-my-way': () => require('../find-my-way'),
   'fs': () => require('../fs'),
@@ -68,7 +69,7 @@ module.exports = {
   'mocha': () => require('../mocha'),
   'mocha-each': () => require('../mocha'),
   'moleculer': () => require('../moleculer'),
-  'mongodb': () => require('../mongodb-core'),
+  'mongodb': () => require('../mongodb'),
   'mongodb-core': () => require('../mongodb-core'),
   'mongoose': () => require('../mongoose'),
   'mysql': () => require('../mysql'),

package/packages/datadog-instrumentations/src/jest.js

@@ -46,6 +46,8 @@ let isCodeCoverageEnabled = false
 let isSuitesSkippingEnabled = false
 let isSuitesSkipped = false
 let numSkippedSuites = 0
+let hasUnskippableSuites = false
+let hasForcedToRunSuites = false
 
 const sessionAsyncResource = new AsyncResource('bound-anonymous-fn')
 
@@ -205,15 +207,22 @@ addHook({
     const [test] = shardedTests
     const rootDir = test && test.context && test.context.config && test.context.config.rootDir
 
-    const { skippedSuites, suitesToRun } = getJestSuitesToRun(skippableSuites, shardedTests, rootDir || process.cwd())
+    const jestSuitesToRun = getJestSuitesToRun(skippableSuites, shardedTests, rootDir || process.cwd())
 
-    isSuitesSkipped = suitesToRun.length !== shardedTests.length
-    numSkippedSuites = skippedSuites.length
+    log.debug(
+      () => `${jestSuitesToRun.suitesToRun.length} out of ${shardedTests.length} suites are going to run.`
+    )
 
-    itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion })
+    hasUnskippableSuites = jestSuitesToRun.hasUnskippableSuites
+    hasForcedToRunSuites = jestSuitesToRun.hasForcedToRunSuites
+
+    isSuitesSkipped = jestSuitesToRun.suitesToRun.length !== shardedTests.length
+    numSkippedSuites = jestSuitesToRun.skippedSuites.length
+
+    itrSkippedSuitesCh.publish({ skippedSuites: jestSuitesToRun.skippedSuites, frameworkVersion })
 
     skippableSuites = []
-    return suitesToRun
+    return jestSuitesToRun.suitesToRun
   })
   return sequencerPackage
 })
@@ -268,7 +277,16 @@ function cliWrapper (cli, jestVersion) {
 
     const result = await runCLI.apply(this, arguments)
 
-    const { results: { success, coverageMap } } = result
+    const {
+      results: {
+        success,
+        coverageMap,
+        numFailedTestSuites,
+        numFailedTests,
+        numTotalTests,
+        numTotalTestSuites
+      }
+    } = result
 
     let testCodeCoverageLinesTotal
     try {
@@ -277,15 +295,30 @@ function cliWrapper (cli, jestVersion) {
     } catch (e) {
       // ignore errors
     }
+    let status, error
+
+    if (success) {
+      if (numTotalTests === 0 && numTotalTestSuites === 0) {
+        status = 'skip'
+      } else {
+        status = 'pass'
+      }
+    } else {
+      status = 'fail'
+      error = new Error(`Failed test suites: ${numFailedTestSuites}. Failed tests: ${numFailedTests}`)
+    }
 
     sessionAsyncResource.runInAsyncScope(() => {
       testSessionFinishCh.publish({
-        status: success ? 'pass' : 'fail',
+        status,
         isSuitesSkipped,
         isSuitesSkippingEnabled,
         isCodeCoverageEnabled,
         testCodeCoverageLinesTotal,
-        numSkippedSuites
+        numSkippedSuites,
+        hasUnskippableSuites,
+        hasForcedToRunSuites,
+        error
       })
     })
 
@@ -357,23 +390,23 @@ function jestAdapterWrapper (jestAdapter, jestVersion) {
           status = 'fail'
         }
 
-        const coverageFiles = getCoveredFilenamesFromCoverage(environment.global.__coverage__)
-          .map(filename => getTestSuitePath(filename, environment.rootDir))
-
         /**
          * Child processes do not each request ITR configuration, so the jest's parent process
          * needs to pass them the configuration. This is done via _ddTestCodeCoverageEnabled, which
          * controls whether coverage is reported.
-         */
-        if (coverageFiles &&
-          environment.testEnvironmentOptions &&
-          environment.testEnvironmentOptions._ddTestCodeCoverageEnabled) {
+        */
+        if (environment.testEnvironmentOptions?._ddTestCodeCoverageEnabled) {
+          const coverageFiles = getCoveredFilenamesFromCoverage(environment.global.__coverage__)
+            .map(filename => getTestSuitePath(filename, environment.rootDir))
           asyncResource.runInAsyncScope(() => {
             testSuiteCodeCoverageCh.publish([...coverageFiles, environment.testSuite])
           })
         }
         testSuiteFinishCh.publish({ status, errorMessage })
         return suiteResults
+      }).catch(error => {
+        testSuiteFinishCh.publish({ status: 'fail', error })
+        throw error
       })
     })
   })
@@ -500,16 +533,21 @@ addHook({
     const testPaths = await getTestPaths.apply(this, arguments)
     const { tests } = testPaths
 
-    const { skippedSuites, suitesToRun } = getJestSuitesToRun(skippableSuites, tests, rootDir)
+    const jestSuitesToRun = getJestSuitesToRun(skippableSuites, tests, rootDir)
+
+    log.debug(() => `${jestSuitesToRun.suitesToRun.length} out of ${tests.length} suites are going to run.`)
+
+    hasUnskippableSuites = jestSuitesToRun.hasUnskippableSuites
+    hasForcedToRunSuites = jestSuitesToRun.hasForcedToRunSuites
 
-    isSuitesSkipped = suitesToRun.length !== tests.length
-    numSkippedSuites = skippedSuites.length
+    isSuitesSkipped = jestSuitesToRun.suitesToRun.length !== tests.length
+    numSkippedSuites = jestSuitesToRun.skippedSuites.length
 
-    itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion })
+    itrSkippedSuitesCh.publish({ skippedSuites: jestSuitesToRun.skippedSuites, frameworkVersion })
 
     skippableSuites = []
 
-    return { ...testPaths, tests: suitesToRun }
+    return { ...testPaths, tests: jestSuitesToRun.suitesToRun }
   })
 
   return searchSourcePackage

package/packages/datadog-instrumentations/src/knex.js

@@ -1,9 +1,12 @@
 'use strict'
 
-const { addHook } = require('./helpers/instrument')
+const { addHook, channel, AsyncResource } = require('./helpers/instrument')
 const { wrapThen } = require('./helpers/promise')
 const shimmer = require('../../datadog-shimmer')
 
+const startRawQueryCh = channel('datadog:knex:raw:start')
+const finishRawQueryCh = channel('datadog:knex:raw:finish')
+
 patch('lib/query/builder.js')
 patch('lib/raw.js')
 patch('lib/schema/builder.js')
@@ -18,3 +21,68 @@ function patch (file) {
     return Builder
   })
 }
+
+addHook({
+  name: 'knex',
+  versions: ['>=2'],
+  file: 'lib/knex-builder/Knex.js'
+}, Knex => {
+  shimmer.wrap(Knex.Client.prototype, 'raw', raw => function () {
+    if (!startRawQueryCh.hasSubscribers) {
+      return raw.apply(this, arguments)
+    }
+
+    const sql = arguments[0]
+
+    // Skip query done by Knex to get the value used for undefined
+    if (sql === 'DEFAULT') {
+      return raw.apply(this, arguments)
+    }
+
+    const asyncResource = new AsyncResource('bound-anonymous-fn')
+
+    function finish () {
+      finishRawQueryCh.publish()
+    }
+
+    return asyncResource.runInAsyncScope(() => {
+      startRawQueryCh.publish({ sql, dialect: this.dialect })
+
+      const rawResult = raw.apply(this, arguments)
+      shimmer.wrap(rawResult, 'then', originalThen => function () {
+        return asyncResource.runInAsyncScope(() => {
+          arguments[0] = wrapCallbackWithFinish(arguments[0], finish)
+          if (arguments[1]) arguments[1] = wrapCallbackWithFinish(arguments[1], finish)
+
+          const originalThenResult = originalThen.apply(this, arguments)
+
+          shimmer.wrap(originalThenResult, 'catch', originalCatch => function () {
+            arguments[0] = wrapCallbackWithFinish(arguments[0], finish)
+            return originalCatch.apply(this, arguments)
+          })
+
+          return originalThenResult
+        })
+      })
+
+      shimmer.wrap(rawResult, 'asCallback', originalAsCallback => function () {
+        return asyncResource.runInAsyncScope(() => {
+          arguments[0] = wrapCallbackWithFinish(arguments[0], finish)
+          return originalAsCallback.apply(this, arguments)
+        })
+      })
+
+      return rawResult
+    })
+  })
+  return Knex
+})
+
+function wrapCallbackWithFinish (callback, finish) {
+  if (typeof callback !== 'function') return callback
+
+  return function () {
+    finish()
+    callback.apply(this, arguments)
+  }
+}

package/packages/datadog-instrumentations/src/mocha.js

@@ -1,9 +1,10 @@
 const { createCoverageMap } = require('istanbul-lib-coverage')
 
+const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util')
+
 const { addHook, channel, AsyncResource } = require('./helpers/instrument')
 const shimmer = require('../../datadog-shimmer')
 const log = require('../../dd-trace/src/log')
-
 const {
   getCoveredFilenamesFromCoverage,
   resetCoverage,
@@ -50,6 +51,8 @@ let suitesToSkip = []
 let frameworkVersion
 let isSuitesSkipped = false
 let skippedSuites = []
+const unskippableSuites = []
+let isForcedToRun = false
 
 function getSuitesByTestFile (root) {
   const suitesByTestFile = {}
@@ -104,7 +107,8 @@ function getFilteredSuites (originalSuites) {
   return originalSuites.reduce((acc, suite) => {
     const testPath = getTestSuitePath(suite.file, process.cwd())
     const shouldSkip = suitesToSkip.includes(testPath)
-    if (shouldSkip) {
+    const isUnskippable = unskippableSuites.includes(suite.file)
+    if (shouldSkip && !isUnskippable) {
       acc.skippedSuites.add(testPath)
     } else {
       acc.suitesToRun.push(suite)
@@ -129,11 +133,20 @@ function mochaHook (Runner) {
 
     this.once('end', testRunAsyncResource.bind(function () {
       let status = 'pass'
+      let error
       if (this.stats) {
         status = this.stats.failures === 0 ? 'pass' : 'fail'
+        if (this.stats.tests === 0) {
+          status = 'skip'
+        }
       } else if (this.failures !== 0) {
         status = 'fail'
       }
+
+      if (status === 'fail') {
+        error = new Error(`Failed tests: ${this.failures}.`)
+      }
+
       testFileToSuiteAr.clear()
 
       let testCodeCoverageLinesTotal
@@ -151,7 +164,10 @@ function mochaHook (Runner) {
         status,
         isSuitesSkipped,
         testCodeCoverageLinesTotal,
-        numSkippedSuites: skippedSuites.length
+        numSkippedSuites: skippedSuites.length,
+        hasForcedToRunSuites: isForcedToRun,
+        hasUnskippableSuites: !!unskippableSuites.length,
+        error
       })
     }))
 
@@ -172,8 +188,10 @@ function mochaHook (Runner) {
       if (!asyncResource) {
         asyncResource = new AsyncResource('bound-anonymous-fn')
         testFileToSuiteAr.set(suite.file, asyncResource)
+        const isUnskippable = unskippableSuites.includes(suite.file)
+        isForcedToRun = isUnskippable && suitesToSkip.includes(getTestSuitePath(suite.file, process.cwd()))
         asyncResource.runInAsyncScope(() => {
-          testSuiteStartCh.publish(suite)
+          testSuiteStartCh.publish({ testSuite: suite.file, isUnskippable, isForcedToRun })
         })
       }
     })
@@ -370,6 +388,13 @@ addHook({
 
     const runner = run.apply(this, arguments)
 
+    this.files.forEach(path => {
+      const isUnskippable = isMarkedAsUnskippable({ path })
+      if (isUnskippable) {
+        unskippableSuites.push(path)
+      }
+    })
+
     const onReceivedSkippableSuites = ({ err, skippableSuites }) => {
       if (err) {
         suitesToSkip = []
@@ -381,6 +406,11 @@ addHook({
       const { suitesToRun } = filteredSuites
 
       isSuitesSkipped = suitesToRun.length !== runner.suite.suites.length
+
+      log.debug(
+        () => `${suitesToRun.length} out of ${runner.suite.suites.length} suites are going to run.`
+      )
+
       runner.suite.suites = suitesToRun
 
       skippedSuites = Array.from(filteredSuites.skippedSuites)

package/packages/datadog-instrumentations/src/mongodb.js

@@ -0,0 +1,63 @@
+'use strict'
+
+require('./mongodb-core')
+
+const {
+  channel,
+  addHook,
+  AsyncResource
+} = require('./helpers/instrument')
+const shimmer = require('../../datadog-shimmer')
+
+// collection methods with filter
+const collectionMethodsWithFilter = [
+  'count',
+  'countDocuments',
+  'deleteMany',
+  'deleteOne',
+  'find',
+  'findOneAndDelete',
+  'findOneAndReplace',
+  'replaceOne'
+] // findOne is ignored because it calls to find
+
+const collectionMethodsWithTwoFilters = [
+  'findOneAndUpdate',
+  'updateMany',
+  'updateOne'
+]
+
+const startCh = channel('datadog:mongodb:collection:filter:start')
+
+addHook({ name: 'mongodb', versions: ['>=3.3 <5', '5', '>=6'] }, mongodb => {
+  [...collectionMethodsWithFilter, ...collectionMethodsWithTwoFilters].forEach(methodName => {
+    if (!(methodName in mongodb.Collection.prototype)) return
+
+    const useTwoArguments = collectionMethodsWithTwoFilters.includes(methodName)
+
+    shimmer.wrap(mongodb.Collection.prototype, methodName, method => {
+      return function () {
+        if (!startCh.hasSubscribers) {
+          return method.apply(this, arguments)
+        }
+
+        const asyncResource = new AsyncResource('bound-anonymous-fn')
+
+        return asyncResource.runInAsyncScope(() => {
+          const filters = [arguments[0]]
+          if (useTwoArguments) {
+            filters.push(arguments[1])
+          }
+
+          startCh.publish({
+            filters,
+            methodName
+          })
+
+          return method.apply(this, arguments)
+        })
+      }
+    })
+  })
+  return mongodb
+})