dd-trace 4.18.0 → 5.6.0

package/packages/datadog-instrumentations/src/kafkajs.js

@@ -8,13 +8,31 @@ const {
 const shimmer = require('../../datadog-shimmer')
 
 const producerStartCh = channel('apm:kafkajs:produce:start')
+const producerCommitCh = channel('apm:kafkajs:produce:commit')
 const producerFinishCh = channel('apm:kafkajs:produce:finish')
 const producerErrorCh = channel('apm:kafkajs:produce:error')
 
 const consumerStartCh = channel('apm:kafkajs:consume:start')
+const consumerCommitCh = channel('apm:kafkajs:consume:commit')
 const consumerFinishCh = channel('apm:kafkajs:consume:finish')
 const consumerErrorCh = channel('apm:kafkajs:consume:error')
 
+function commitsFromEvent (event) {
+  const { payload: { groupId, topics } } = event
+  const commitList = []
+  for (const { topic, partitions } of topics) {
+    for (const { partition, offset } of partitions) {
+      commitList.push({
+        groupId,
+        partition,
+        offset,
+        topic
+      })
+    }
+  }
+  consumerCommitCh.publish(commitList)
+}
+
 addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKafka) => {
   class Kafka extends BaseKafka {
     constructor (options) {
@@ -58,6 +76,12 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf
             })
           )
 
+          result.then(res => {
+            if (producerCommitCh.hasSubscribers) {
+              producerCommitCh.publish(res)
+            }
+          })
+
           return result
         } catch (e) {
           producerErrorCh.publish(e)
@@ -75,6 +99,9 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf
     }
 
     const consumer = createConsumer.apply(this, arguments)
+
+    consumer.on(consumer.events.COMMIT_OFFSETS, commitsFromEvent)
+
     const run = consumer.run
 
     const groupId = arguments[0].groupId

package/packages/datadog-instrumentations/src/mocha.js

@@ -11,7 +11,9 @@ const {
   mergeCoverage,
   getTestSuitePath,
   fromCoverageMapToCoverage,
-  getCallSites
+  getCallSites,
+  addEfdStringToTestName,
+  removeEfdStringFromTestName
 } = require('../../dd-trace/src/plugins/util/test')
 
 const testStartCh = channel('ci:mocha:test:start')
@@ -20,7 +22,8 @@ const skipCh = channel('ci:mocha:test:skip')
 const testFinishCh = channel('ci:mocha:test:finish')
 const parameterizedTestCh = channel('ci:mocha:test:parameterize')
 
-const itrConfigurationCh = channel('ci:mocha:itr-configuration')
+const libraryConfigurationCh = channel('ci:mocha:library-configuration')
+const knownTestsCh = channel('ci:mocha:known-tests')
 const skippableSuitesCh = channel('ci:mocha:test-suite:skippable')
 
 const testSessionStartCh = channel('ci:mocha:session:start')
@@ -40,6 +43,7 @@ const testToAr = new WeakMap()
 const originalFns = new WeakMap()
 const testFileToSuiteAr = new Map()
 const testToStartLine = new WeakMap()
+const newTests = {}
 
 // `isWorker` is true if it's a Mocha worker
 let isWorker = false
@@ -53,6 +57,11 @@ let isSuitesSkipped = false
 let skippedSuites = []
 const unskippableSuites = []
 let isForcedToRun = false
+let itrCorrelationId = ''
+let isEarlyFlakeDetectionEnabled = false
+let earlyFlakeDetectionNumRetries = 0
+let isSuitesSkippingEnabled = false
+let knownTests = []
 
 function getSuitesByTestFile (root) {
   const suitesByTestFile = {}
@@ -92,6 +101,29 @@ function isRetry (test) {
   return test._currentRetry !== undefined && test._currentRetry !== 0
 }
 
+function getTestFullName (test) {
+  return `mocha.${getTestSuitePath(test.file, process.cwd())}.${removeEfdStringFromTestName(test.fullTitle())}`
+}
+
+function isNewTest (test) {
+  const testSuite = getTestSuitePath(test.file, process.cwd())
+  const testName = removeEfdStringFromTestName(test.fullTitle())
+  const testsForSuite = knownTests.mocha?.[testSuite] || []
+  return !testsForSuite.includes(testName)
+}
+
+function retryTest (test) {
+  const originalTestName = test.title
+  const suite = test.parent
+  for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) {
+    const clonedTest = test.clone()
+    clonedTest.title = addEfdStringToTestName(originalTestName, retryIndex + 1)
+    suite.addTest(clonedTest)
+    clonedTest._ddIsNew = true
+    clonedTest._ddIsEfdRetry = true
+  }
+}
+
 function getTestAsyncResource (test) {
   if (!test.fn) {
     return testToAr.get(test)
@@ -122,6 +154,19 @@ function mochaHook (Runner) {
 
   patched.add(Runner)
 
+  shimmer.wrap(Runner.prototype, 'runTests', runTests => function (suite, fn) {
+    if (isEarlyFlakeDetectionEnabled) {
+      // by the time we reach `this.on('test')`, it is too late. We need to add retries here
+      suite.tests.forEach(test => {
+        if (!test.isPending() && isNewTest(test)) {
+          test._ddIsNew = true
+          retryTest(test)
+        }
+      })
+    }
+    return runTests.apply(this, arguments)
+  })
+
   shimmer.wrap(Runner.prototype, 'run', run => function () {
     if (!testStartCh.hasSubscribers || isWorker) {
       return run.apply(this, arguments)
@@ -143,6 +188,24 @@ function mochaHook (Runner) {
         status = 'fail'
       }
 
+      if (isEarlyFlakeDetectionEnabled) {
+        /**
+         * If Early Flake Detection (EFD) is enabled the logic is as follows:
+         * - If all attempts for a test are failing, the test has failed and we will let the test process fail.
+         * - If just a single attempt passes, we will prevent the test process from failing.
+         * The rationale behind is the following: you may still be able to block your CI pipeline by gating
+         * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too.
+         */
+        for (const tests of Object.values(newTests)) {
+          const failingNewTests = tests.filter(test => test.isFailed())
+          const areAllNewTestsFailing = failingNewTests.length === tests.length
+          if (failingNewTests.length && !areAllNewTestsFailing) {
+            this.stats.failures -= failingNewTests.length
+            this.failures -= failingNewTests.length
+          }
+        }
+      }
+
       if (status === 'fail') {
         error = new Error(`Failed tests: ${this.failures}.`)
       }
@@ -167,7 +230,8 @@ function mochaHook (Runner) {
         numSkippedSuites: skippedSuites.length,
         hasForcedToRunSuites: isForcedToRun,
         hasUnskippableSuites: !!unskippableSuites.length,
-        error
+        error,
+        isEarlyFlakeDetectionEnabled
       })
     }))
 
@@ -191,7 +255,12 @@ function mochaHook (Runner) {
         const isUnskippable = unskippableSuites.includes(suite.file)
         isForcedToRun = isUnskippable && suitesToSkip.includes(getTestSuitePath(suite.file, process.cwd()))
         asyncResource.runInAsyncScope(() => {
-          testSuiteStartCh.publish({ testSuite: suite.file, isUnskippable, isForcedToRun })
+          testSuiteStartCh.publish({
+            testSuite: suite.file,
+            isUnskippable,
+            isForcedToRun,
+            itrCorrelationId
+          })
         })
       }
     })
@@ -247,8 +316,35 @@ function mochaHook (Runner) {
       const testStartLine = testToStartLine.get(test)
       const asyncResource = new AsyncResource('bound-anonymous-fn')
       testToAr.set(test.fn, asyncResource)
+
+      const {
+        file: testSuiteAbsolutePath,
+        title,
+        _ddIsNew: isNew,
+        _ddIsEfdRetry: isEfdRetry
+      } = test
+
+      const testInfo = {
+        testName: test.fullTitle(),
+        testSuiteAbsolutePath,
+        title,
+        isNew,
+        isEfdRetry,
+        testStartLine
+      }
+
+      // We want to store the result of the new tests
+      if (isNew) {
+        const testFullName = getTestFullName(test)
+        if (newTests[testFullName]) {
+          newTests[testFullName].push(test)
+        } else {
+          newTests[testFullName] = [test]
+        }
+      }
+
       asyncResource.runInAsyncScope(() => {
-        testStartCh.publish({ test, testStartLine })
+        testStartCh.publish(testInfo)
       })
     })
 
@@ -317,10 +413,23 @@ function mochaHook (Runner) {
     })
 
     this.on('pending', (test) => {
+      const testStartLine = testToStartLine.get(test)
+      const {
+        file: testSuiteAbsolutePath,
+        title
+      } = test
+
+      const testInfo = {
+        testName: test.fullTitle(),
+        testSuiteAbsolutePath,
+        title,
+        testStartLine
+      }
+
       const asyncResource = getTestAsyncResource(test)
       if (asyncResource) {
         asyncResource.runInAsyncScope(() => {
-          skipCh.publish(test)
+          skipCh.publish(testInfo)
         })
       } else {
         // if there is no async resource, the test has been skipped through `test.skip`
@@ -332,7 +441,7 @@ function mochaHook (Runner) {
           testToAr.set(test, skippedTestAsyncResource)
         }
         skippedTestAsyncResource.runInAsyncScope(() => {
-          skipCh.publish(test)
+          skipCh.publish(testInfo)
         })
       }
     })
@@ -352,8 +461,8 @@ function mochaEachHook (mochaEach) {
     const [params] = arguments
     const { it, ...rest } = mochaEach.apply(this, arguments)
     return {
-      it: function (name) {
-        parameterizedTestCh.publish({ name, params })
+      it: function (title) {
+        parameterizedTestCh.publish({ title, params })
         it.apply(this, arguments)
       },
       ...rest
@@ -378,7 +487,7 @@ addHook({
       return run.apply(this, arguments)
     }
 
-    if (!itrConfigurationCh.hasSubscribers || this.isWorker) {
+    if (!libraryConfigurationCh.hasSubscribers || this.isWorker) {
       if (this.isWorker) {
         isWorker = true
       }
@@ -395,11 +504,12 @@ addHook({
       }
     })
 
-    const onReceivedSkippableSuites = ({ err, skippableSuites }) => {
+    const onReceivedSkippableSuites = ({ err, skippableSuites, itrCorrelationId: responseItrCorrelationId }) => {
       if (err) {
         suitesToSkip = []
       } else {
         suitesToSkip = skippableSuites
+        itrCorrelationId = responseItrCorrelationId
       }
       // We remove the suites that we skip through ITR
       const filteredSuites = getFilteredSuites(runner.suite.suites)
@@ -418,21 +528,47 @@ addHook({
       global.run()
     }
 
-    const onReceivedConfiguration = ({ err }) => {
+    const onReceivedKnownTests = ({ err, knownTests: receivedKnownTests }) => {
       if (err) {
-        return global.run()
+        knownTests = []
+        isEarlyFlakeDetectionEnabled = false
+      } else {
+        knownTests = receivedKnownTests
+      }
+
+      if (isSuitesSkippingEnabled) {
+        skippableSuitesCh.publish({
+          onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites)
+        })
+      } else {
+        global.run()
       }
-      if (!skippableSuitesCh.hasSubscribers) {
+    }
+
+    const onReceivedConfiguration = ({ err, libraryConfig }) => {
+      if (err || !skippableSuitesCh.hasSubscribers || !knownTestsCh.hasSubscribers) {
         return global.run()
       }
 
-      skippableSuitesCh.publish({
-        onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites)
-      })
+      isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled
+      isSuitesSkippingEnabled = libraryConfig.isSuitesSkippingEnabled
+      earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries
+
+      if (isEarlyFlakeDetectionEnabled) {
+        knownTestsCh.publish({
+          onDone: mochaRunAsyncResource.bind(onReceivedKnownTests)
+        })
+      } else if (isSuitesSkippingEnabled) {
+        skippableSuitesCh.publish({
+          onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites)
+        })
+      } else {
+        global.run()
+      }
     }
 
     mochaRunAsyncResource.runInAsyncScope(() => {
-      itrConfigurationCh.publish({
+      libraryConfigurationCh.publish({
         onDone: mochaRunAsyncResource.bind(onReceivedConfiguration)
       })
     })

package/packages/datadog-instrumentations/src/mongodb-core.js

@@ -32,12 +32,18 @@ addHook({ name: 'mongodb', versions: ['>=4 <4.6.0'], file: 'lib/cmap/connection.
   return Connection
 })
 
-addHook({ name: 'mongodb', versions: ['>=4.6.0'], file: 'lib/cmap/connection.js' }, Connection => {
+addHook({ name: 'mongodb', versions: ['>=4.6.0 <6.4.0'], file: 'lib/cmap/connection.js' }, Connection => {
   const proto = Connection.Connection.prototype
   shimmer.wrap(proto, 'command', command => wrapConnectionCommand(command, 'command'))
   return Connection
 })
 
+addHook({ name: 'mongodb', versions: ['>=6.4.0'], file: 'lib/cmap/connection.js' }, Connection => {
+  const proto = Connection.Connection.prototype
+  shimmer.wrap(proto, 'command', command => wrapConnectionCommand(command, 'command', undefined, instrumentPromise))
+  return Connection
+})
+
 addHook({ name: 'mongodb', versions: ['>=3.3 <4'], file: 'lib/core/wireprotocol/index.js' }, wp => wrapWp(wp))
 
 addHook({ name: 'mongodb-core', versions: ['>=3.2'], file: 'lib/wireprotocol/index.js' }, wp => wrapWp(wp))
@@ -89,7 +95,7 @@ function wrapUnifiedCommand (command, operation, name) {
   return shimmer.wrap(command, wrapped)
 }
 
-function wrapConnectionCommand (command, operation, name) {
+function wrapConnectionCommand (command, operation, name, instrumentFn = instrument) {
   const wrapped = function (ns, ops) {
     if (!startCh.hasSubscribers) {
       return command.apply(this, arguments)
@@ -101,7 +107,7 @@ function wrapConnectionCommand (command, operation, name) {
     const topology = { s: { options } }
 
     ns = `${ns.db}.${ns.collection}`
-    return instrument(operation, command, this, arguments, topology, ns, ops, { name })
+    return instrumentFn(operation, command, this, arguments, topology, ns, ops, { name })
   }
   return shimmer.wrap(command, wrapped)
 }
@@ -179,3 +185,28 @@ function instrument (operation, command, ctx, args, server, ns, ops, options = {
     }
   })
 }
+
+function instrumentPromise (operation, command, ctx, args, server, ns, ops, options = {}) {
+  const name = options.name || (ops && Object.keys(ops)[0])
+
+  const serverInfo = server && server.s && server.s.options
+  const asyncResource = new AsyncResource('bound-anonymous-fn')
+
+  return asyncResource.runInAsyncScope(() => {
+    startCh.publish({ ns, ops, options: serverInfo, name })
+
+    const promise = command.apply(ctx, args)
+
+    promise.then(function (res) {
+      finishCh.publish()
+      return res
+    }, function (err) {
+      errorCh.publish(err)
+      finishCh.publish()
+
+      return Promise.reject(err)
+    })
+
+    return promise
+  })
+}

package/packages/datadog-instrumentations/src/mongoose.js

@@ -79,21 +79,26 @@ addHook({
         })
 
         let callbackWrapped = false
-        const lastArgumentIndex = arguments.length - 1
 
-        if (typeof arguments[lastArgumentIndex] === 'function') {
-          // is a callback, wrap it to execute finish()
-          shimmer.wrap(arguments, lastArgumentIndex, originalCb => {
-            return function () {
-              finish()
+        const wrapCallbackIfExist = (args) => {
+          const lastArgumentIndex = args.length - 1
 
-              return originalCb.apply(this, arguments)
-            }
-          })
+          if (typeof args[lastArgumentIndex] === 'function') {
+            // is a callback, wrap it to execute finish()
+            shimmer.wrap(args, lastArgumentIndex, originalCb => {
+              return function () {
+                finish()
+
+                return originalCb.apply(this, arguments)
+              }
+            })
 
-          callbackWrapped = true
+            callbackWrapped = true
+          }
         }
 
+        wrapCallbackIfExist(arguments)
+
         return asyncResource.runInAsyncScope(() => {
           startCh.publish({
             filters,
@@ -106,8 +111,16 @@ addHook({
           if (!callbackWrapped) {
             shimmer.wrap(res, 'exec', originalExec => {
               return function wrappedExec () {
+                if (!callbackWrapped) {
+                  wrapCallbackIfExist(arguments)
+                }
+
                 const execResult = originalExec.apply(this, arguments)
 
+                if (callbackWrapped || typeof execResult?.then !== 'function') {
+                  return execResult
+                }
+
                 // wrap them method, wrap resolve and reject methods
                 shimmer.wrap(execResult, 'then', originalThen => {
                   return function wrappedThen () {

package/packages/datadog-instrumentations/src/mquery.js

@@ -0,0 +1,65 @@
+'use strict'
+
+const dc = require('dc-polyfill')
+const {
+  channel,
+  addHook
+} = require('./helpers/instrument')
+const shimmer = require('../../datadog-shimmer')
+
+const prepareCh = channel('datadog:mquery:filter:prepare')
+const tracingCh = dc.tracingChannel('datadog:mquery:filter')
+
+const methods = [
+  'find',
+  'findOne',
+  'findOneAndRemove',
+  'findOneAndDelete',
+  'count',
+  'distinct',
+  'where'
+]
+
+const methodsOptionalArgs = ['findOneAndUpdate']
+
+function getFilters (args, methodName) {
+  const [arg0, arg1] = args
+
+  const filters = arg0 && typeof arg0 === 'object' ? [arg0] : []
+
+  if (arg1 && typeof arg1 === 'object' && methodsOptionalArgs.includes(methodName)) {
+    filters.push(arg1)
+  }
+
+  return filters
+}
+
+addHook({
+  name: 'mquery',
+  versions: ['>=5.0.0']
+}, Query => {
+  [...methods, ...methodsOptionalArgs].forEach(methodName => {
+    if (!(methodName in Query.prototype)) return
+
+    shimmer.wrap(Query.prototype, methodName, method => {
+      return function () {
+        if (prepareCh.hasSubscribers) {
+          const filters = getFilters(arguments, methodName)
+          if (filters?.length) {
+            prepareCh.publish({ filters })
+          }
+        }
+
+        return method.apply(this, arguments)
+      }
+    })
+  })
+
+  shimmer.wrap(Query.prototype, 'exec', originalExec => {
+    return function wrappedExec () {
+      return tracingCh.tracePromise(originalExec, {}, this, arguments)
+    }
+  })
+
+  return Query
+})

package/packages/datadog-instrumentations/src/net.js

@@ -17,8 +17,16 @@ const errorTCPCh = channel('apm:net:tcp:error')
 
 const connectionCh = channel(`apm:net:tcp:connection`)
 
-addHook({ name: 'net' }, net => {
-  require('dns')
+const names = ['net', 'node:net']
+
+addHook({ name: names }, (net, version, name) => {
+  // explicitly require dns so that net gets an instrumented instance
+  // so that we don't miss the dns calls
+  if (name === 'net') {
+    require('dns')
+  } else {
+    require('node:dns')
+  }
 
   shimmer.wrap(net.Socket.prototype, 'connect', connect => function () {
     if (!startICPCh.hasSubscribers || !startTCPCh.hasSubscribers) {

package/packages/datadog-instrumentations/src/next.js

@@ -65,7 +65,7 @@ function wrapRenderToHTML (renderToHTML) {
 
 function wrapRenderErrorToHTML (renderErrorToHTML) {
   return function (err, req, res, pathname, query) {
-    return instrument(req, res, () => renderErrorToHTML.apply(this, arguments))
+    return instrument(req, res, err, () => renderErrorToHTML.apply(this, arguments))
   }
 }
 
@@ -76,8 +76,8 @@ function wrapRenderToResponse (renderToResponse) {
 }
 
 function wrapRenderErrorToResponse (renderErrorToResponse) {
-  return function (ctx) {
-    return instrument(ctx.req, ctx.res, () => renderErrorToResponse.apply(this, arguments))
+  return function (ctx, err) {
+    return instrument(ctx.req, ctx.res, err, () => renderErrorToResponse.apply(this, arguments))
   }
 }
 
@@ -111,13 +111,23 @@ function getPageFromPath (page, dynamicRoutes = []) {
   return getPagePath(page)
 }
 
-function instrument (req, res, handler) {
+function instrument (req, res, error, handler) {
+  if (typeof error === 'function') {
+    handler = error
+    error = null
+  }
+
   req = req.originalRequest || req
   res = res.originalResponse || res
 
   // TODO support middleware properly in the future?
   const isMiddleware = req.headers[MIDDLEWARE_HEADER]
-  if (isMiddleware || requests.has(req)) return handler()
+  if (isMiddleware || requests.has(req)) {
+    if (error) {
+      errorChannel.publish({ error })
+    }
+    return handler()
+  }
 
   requests.add(req)
 
@@ -144,7 +154,9 @@ function instrument (req, res, handler) {
 function wrapServeStatic (serveStatic) {
   return function (req, res, path) {
     return instrument(req, res, () => {
-      if (pageLoadChannel.hasSubscribers && path) pageLoadChannel.publish({ page: path })
+      if (pageLoadChannel.hasSubscribers && path) {
+        pageLoadChannel.publish({ page: path, isStatic: true })
+      }
 
       return serveStatic.apply(this, arguments)
     })
@@ -278,9 +290,23 @@ addHook({
   shimmer.massWrap(request.NextRequest.prototype, ['text', 'json'], function (originalMethod) {
     return async function wrappedJson () {
       const body = await originalMethod.apply(this, arguments)
-      bodyParsedChannel.publish({
-        body
-      })
+
+      bodyParsedChannel.publish({ body })
+
+      return body
+    }
+  })
+
+  shimmer.wrap(request.NextRequest.prototype, 'formData', function (originalFormData) {
+    return async function wrappedFormData () {
+      const body = await originalFormData.apply(this, arguments)
+
+      let normalizedBody = body
+      if (typeof body.entries === 'function') {
+        normalizedBody = Object.fromEntries(body.entries())
+      }
+      bodyParsedChannel.publish({ body: normalizedBody })
+
       return body
     }
   })