dd-trace 5.110.0 → 5.112.0

package/packages/datadog-instrumentations/src/mocha/main.js

@@ -36,6 +36,7 @@ const {
   getOnTestEndHandler,
   getOnTestRetryHandler,
   getOnHookEndHandler,
+  patchFailedTestReplayHookUp,
   getOnFailHandler,
   getOnPendingHandler,
   testFileToSuiteCtx,
@@ -382,6 +383,10 @@ function applyTestManagementTestsResponse ({ err, testManagementTests: receivedT
   }
 }
 
+function isFailedTestReplayEnabled () {
+  return config.isTestDynamicInstrumentationEnabled && config.isDiEnabled
+}
+
 function getExecutionConfiguration (runner, isParallel, frameworkVersion, onFinishRequest, localSuites) {
   const ctx = {
     isParallel,
@@ -471,7 +476,7 @@ function getExecutionConfiguration (runner, isParallel, frameworkVersion, onFini
     }
   }
 
-  const onReceivedConfiguration = ({ err, libraryConfig, repositoryRoot }) => {
+  const onReceivedConfiguration = ({ err, isTestDynamicInstrumentationEnabled, libraryConfig, repositoryRoot }) => {
     if (err || !skippableSuitesCh.hasSubscribers || !knownTestsCh.hasSubscribers) {
       return mochaGlobalRunCh.runStores(ctx, () => {
         onFinishRequest()
@@ -492,6 +497,8 @@ function getExecutionConfiguration (runner, isParallel, frameworkVersion, onFini
     config.isSuitesSkippingEnabled = config.isItrEnabled && libraryConfig.isSuitesSkippingEnabled
     config.isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled
     config.flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount
+    config.isDiEnabled = libraryConfig.isDiEnabled
+    config.isTestDynamicInstrumentationEnabled = isTestDynamicInstrumentationEnabled
 
     getTestOptimizationRequestResults({
       isKnownTestsEnabled: config.isKnownTestsEnabled,
@@ -555,6 +562,9 @@ addHook({
     })
 
     getExecutionConfiguration(runner, false, frameworkVersion, () => {
+      if (isFailedTestReplayEnabled()) {
+        patchFailedTestReplayHookUp(runner.constructor)
+      }
       if (config.isKnownTestsEnabled) {
         const testSuites = this.files.map(file => getTestSuitePath(file, process.cwd()))
         const isFaulty = getIsFaultyEarlyFlakeDetection(
@@ -780,7 +790,13 @@ addHook({
     this.on('retry', getOnTestRetryHandler(config))
 
     // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted
-    this.on('hook end', getOnHookEndHandler(config))
+    this.on('hook end', getOnHookEndHandler(config, {
+      onStart: incrementPendingRootFinalization,
+      onFinish: function (test) {
+        finishRootSuiteAfterFinalAttempt(test)
+        decrementPendingRootFinalization(test)
+      },
+    }))
 
     this.on('hook end', function (hook) {
       const test = hook.ctx?.currentTest
@@ -1095,7 +1111,8 @@ addHook({
         (!config.isKnownTestsEnabled &&
         !config.isTestManagementTestsEnabled &&
         !config.isImpactedTestsEnabled &&
-        !config.isFlakyTestRetriesEnabled)) {
+        !config.isFlakyTestRetriesEnabled &&
+        !isFailedTestReplayEnabled())) {
       return run.apply(this, arguments)
     }
 
@@ -1146,6 +1163,10 @@ addHook({
       newWorkerArgs._ddFlakyTestRetriesCount = config.flakyTestRetriesCount
     }
 
+    if (isFailedTestReplayEnabled()) {
+      newWorkerArgs._ddIsFailedTestReplayEnabled = true
+    }
+
     // We pass the known tests for the test file to the worker
     const testFileResult = await run.apply(
       this,

package/packages/datadog-instrumentations/src/mocha/utils.js

@@ -2,9 +2,6 @@
 
 const { performance } = require('node:perf_hooks')
 
-// Capture real timers at module load time, before any test can install fake timers.
-const realSetTimeout = setTimeout
-
 const {
   getTestSuitePath,
   DYNAMIC_NAME_RE,
@@ -19,6 +16,7 @@ const shimmer = require('../../../datadog-shimmer')
 // test channels
 const testStartCh = channel('ci:mocha:test:start')
 const testFinishCh = channel('ci:mocha:test:finish')
+const testDiWaitCh = channel('ci:mocha:test:di:wait')
 // after a test has failed, we'll publish to this channel
 const testRetryCh = channel('ci:mocha:test:retry')
 const errorCh = channel('ci:mocha:test:error')
@@ -28,7 +26,6 @@ const isModifiedCh = channel('ci:mocha:test:is-modified')
 // suite channels
 const testSuiteErrorCh = channel('ci:mocha:test-suite:error')
 
-const BREAKPOINT_HIT_GRACE_PERIOD_MS = 200
 const testToContext = new WeakMap()
 const originalFns = new WeakMap()
 const testToStartLine = new WeakMap()
@@ -43,6 +40,12 @@ const testsStatuses = new Map()
 const efdRetryCountByTestFullName = new Map()
 const efdSlowAbortedTests = new Set()
 const attemptToFixExecutions = new Map()
+
+function waitForHitProbe () {
+  const promises = {}
+  testDiWaitCh.publish({ promises })
+  return promises.hitBreakpointPromise
+}
 const loggedAttemptToFixTests = new Set()
 
 function getAfterEachHooks (testOrHook) {
@@ -639,14 +642,8 @@ function getOnTestEndHandler (config, finalAttemptHandlers) {
       finalAttemptHandlers?.onStart?.(test)
     }
 
-    // After finishing it might take a bit for the snapshot to be handled.
-    // This means that tests retried with DI are BREAKPOINT_HIT_GRACE_PERIOD_MS slower at least.
-    if (test._ddShouldWaitForHitProbe || test._retriedTest?._ddShouldWaitForHitProbe) {
-      await new Promise((resolve) => {
-        realSetTimeout(() => {
-          resolve()
-        }, BREAKPOINT_HIT_GRACE_PERIOD_MS)
-      })
+    if (test._retriedTest?._ddShouldWaitForHitProbe) {
+      await waitForHitProbe()
     }
 
     if (shouldFinishTest) {
@@ -665,7 +662,7 @@ function getOnTestEndHandler (config, finalAttemptHandlers) {
   }
 }
 
-function getOnHookEndHandler (config) {
+function getOnHookEndHandler (config, finalAttemptHandlers) {
   return function (hook) {
     const test = hook.ctx.currentTest
     const afterEachHooks = getAfterEachHooks(hook)
@@ -678,22 +675,118 @@ function getOnHookEndHandler (config) {
         // skip to avoid double-publishing
         if (ctx && (!test._ddIsDisabled || test._ddIsAttemptToFix)) {
           const testFinishInfo = getTestFinishInfo(test, status, config, ctx.err || test.err)
-          if (testFinishInfo.finalStatus !== undefined) {
-            test._ddIsFinalAttempt = true
+          const isFinalAttempt = testFinishInfo.finalStatus !== undefined
+          const publishTestFinish = () => {
+            testFinishCh.publish({
+              status,
+              hasBeenRetried: isMochaRetry(test),
+              isLastRetry: getIsLastRetry(test),
+              ...testFinishInfo,
+              ...ctx.currentStore,
+            })
+            if (isFinalAttempt) {
+              test._ddIsFinalAttempt = true
+            }
           }
-          testFinishCh.publish({
-            status,
-            hasBeenRetried: isMochaRetry(test),
-            isLastRetry: getIsLastRetry(test),
-            ...testFinishInfo,
-            ...ctx.currentStore,
-          })
+          if (test._retriedTest?._ddShouldWaitForHitProbe) {
+            if (isFinalAttempt) {
+              finalAttemptHandlers?.onStart?.(test)
+            }
+            test._ddDeferredHookEnd = {
+              waitForHitProbePromise: waitForHitProbe(),
+              publishTestFinish,
+              onFinish: isFinalAttempt ? () => finalAttemptHandlers?.onFinish?.(test) : undefined,
+            }
+            return
+          }
+          publishTestFinish()
         }
       }
     }
   }
 }
 
+function finishDeferredHookEnd (test) {
+  const deferredHookEnd = test?._ddDeferredHookEnd
+  if (!deferredHookEnd) return
+
+  const finish = () => {
+    try {
+      return deferredHookEnd.publishTestFinish()
+    } finally {
+      deferredHookEnd.onFinish?.()
+    }
+  }
+
+  delete test._ddDeferredHookEnd
+  if (!deferredHookEnd.waitForHitProbePromise) return finish()
+
+  return deferredHookEnd.waitForHitProbePromise.then(
+    finish,
+    finish
+  )
+}
+
+/**
+ * Runs a Failed Test Replay hookUp callback after pending DI operations that must happen first.
+ *
+ * @param {(...args: unknown[]) => unknown} fn - Original hookUp completion callback.
+ * @param {object} test - Mocha test currently owning the hook.
+ * @param {Promise<void>|undefined} failedTestReplayPromise - Pending Failed Test Replay wait, if any.
+ * @param {unknown} hookThis - Callback receiver.
+ * @param {IArguments} args - Arguments passed by Mocha.
+ * @returns {unknown}
+ */
+function runFailedTestReplayHookUpCallback (fn, test, failedTestReplayPromise, hookThis, args) {
+  const continueAfterProbe = () => {
+    const deferredHookEndPromise = finishDeferredHookEnd(test)
+    if (deferredHookEndPromise) {
+      return deferredHookEndPromise.then(() => fn.apply(hookThis, args), () => fn.apply(hookThis, args))
+    }
+    return fn.apply(hookThis, args)
+  }
+
+  if (failedTestReplayPromise) {
+    return failedTestReplayPromise.then(continueAfterProbe, continueAfterProbe)
+  }
+  return continueAfterProbe()
+}
+
+/**
+ * Wraps Mocha's hookUp completion callback so retries wait for DI before continuing.
+ *
+ * @param {(...args: unknown[]) => unknown} fn - Original hookUp completion callback.
+ * @param {object} test - Mocha test currently owning the hook.
+ * @param {Promise<void>|undefined} failedTestReplayPromise - Pending Failed Test Replay wait, if any.
+ * @returns {(...args: unknown[]) => unknown}
+ */
+function wrapFailedTestReplayHookUpCallback (fn, test, failedTestReplayPromise) {
+  return shimmer.wrapCallback(fn, fn => function () {
+    return runFailedTestReplayHookUpCallback(fn, test, failedTestReplayPromise, this, arguments)
+  })
+}
+
+const patchedFailedTestReplayHookUp = new WeakSet()
+
+function patchFailedTestReplayHookUp (Runner) {
+  if (patchedFailedTestReplayHookUp.has(Runner)) return
+
+  patchedFailedTestReplayHookUp.add(Runner)
+  shimmer.wrap(Runner.prototype, 'hookUp', hookUp => function (name, fn) {
+    const test = name === 'afterEach' && this.test
+    if (!test) {
+      return hookUp.apply(this, arguments)
+    }
+
+    const failedTestReplayPromise = test._ddFailedTestReplayPromise
+    if (failedTestReplayPromise) {
+      delete test._ddFailedTestReplayPromise
+    }
+
+    return hookUp.call(this, name, wrapFailedTestReplayHookUpCallback(fn, test, failedTestReplayPromise))
+  })
+}
+
 function getOnFailHandler (isMain, config) {
   return function (testOrHook, err) {
     const testFile = testOrHook.file
@@ -764,14 +857,24 @@ function getOnTestRetryHandler (config) {
         config.isFlakyTestRetriesEnabled &&
         !test._ddIsAttemptToFix &&
         !test._ddIsEfdRetry
+      const promises = {}
       testRetryCh.publish({
         isFirstAttempt,
         err,
         willBeRetried,
         test,
         isAtrRetry,
+        promises,
         ...ctx.currentStore,
       })
+      if (promises.setProbePromise && promises.finishTestPromise) {
+        test._ddFailedTestReplayPromise = Promise.all([
+          promises.setProbePromise,
+          promises.finishTestPromise,
+        ]).then(() => {})
+      } else if (promises.setProbePromise || promises.finishTestPromise) {
+        test._ddFailedTestReplayPromise = promises.setProbePromise || promises.finishTestPromise
+      }
     }
     const key = getTestToContextKey(test)
     testToContext.delete(key)
@@ -901,6 +1004,9 @@ module.exports = {
   getOnTestEndHandler,
   getOnTestRetryHandler,
   getOnHookEndHandler,
+  finishDeferredHookEnd,
+  wrapFailedTestReplayHookUpCallback,
+  patchFailedTestReplayHookUp,
   getOnFailHandler,
   getOnPendingHandler,
   testFileToSuiteCtx,

package/packages/datadog-instrumentations/src/mocha/worker.js

@@ -13,6 +13,7 @@ const {
   getOnPendingHandler,
   getOnTestRetryHandler,
   getRunTestsWrapper,
+  patchFailedTestReplayHookUp,
 } = require('./utils')
 require('./common')
 
@@ -22,6 +23,10 @@ const workerFinishCh = channel('ci:mocha:worker:finish')
 
 const config = {}
 
+function isFailedTestReplayEnabled () {
+  return config.isTestDynamicInstrumentationEnabled && config.isDiEnabled
+}
+
 addHook({
   name: 'mocha',
   versions: ['>=8.0.0'],
@@ -60,6 +65,11 @@ addHook({
       delete this.options._ddIsFlakyTestRetriesEnabled
       delete this.options._ddFlakyTestRetriesCount
     }
+    if (this.options._ddIsFailedTestReplayEnabled) {
+      config.isTestDynamicInstrumentationEnabled = true
+      config.isDiEnabled = true
+      delete this.options._ddIsFailedTestReplayEnabled
+    }
     return run.apply(this, args)
   })
 
@@ -78,6 +88,9 @@ addHook({
     if (!workerFinishCh.hasSubscribers) {
       return run.apply(this, args)
     }
+    if (isFailedTestReplayEnabled()) {
+      patchFailedTestReplayHookUp(Runner)
+    }
     // We flush when the worker ends with its test file (a mocha instance in a worker runs a single test file)
     this.once('end', () => {
       workerFinishCh.publish()

package/packages/datadog-instrumentations/src/mongodb.js

@@ -28,6 +28,10 @@ const collectionMethodsWithTwoFilters = [
 
 const startCh = channel('datadog:mongodb:collection:filter:start')
 
+const bulkWriteStartCh = channel('apm:mongodb:bulkwrite:start')
+const bulkWriteFinishCh = channel('apm:mongodb:bulkwrite:finish')
+const bulkWriteErrorCh = channel('apm:mongodb:bulkwrite:error')
+
 addHook({ name: 'mongodb', versions: ['>=3.3 <5', '5', '>=6'] }, mongodb => {
   for (const methodName of [...collectionMethodsWithFilter, ...collectionMethodsWithTwoFilters]) {
     if (!(methodName in mongodb.Collection.prototype)) continue
@@ -55,5 +59,75 @@ addHook({ name: 'mongodb', versions: ['>=3.3 <5', '5', '>=6'] }, mongodb => {
       }
     })
   }
+
+  // `bulkWrite` fans out into separate `insert`/`update`/`delete` wire commands, so wrap
+  // it to open one parent span those per-type commands nest under as children.
+  if ('bulkWrite' in mongodb.Collection.prototype) {
+    shimmer.wrap(mongodb.Collection.prototype, 'bulkWrite', wrapBulkWrite)
+  }
+
   return mongodb
 })
+
+/**
+ * @param {Function} bulkWrite
+ * @returns {Function}
+ */
+function wrapBulkWrite (bulkWrite) {
+  return function (...args) {
+    /* istanbul ignore if: plugin stays subscribed for the whole suite; the disabled fast path is unreachable */
+    if (!bulkWriteStartCh.hasSubscribers) {
+      return bulkWrite.apply(this, args)
+    }
+
+    const ctx = { ns: this.namespace }
+
+    return bulkWriteStartCh.runStores(ctx, () => {
+      // Pre-v5 drivers take a trailing callback and return `undefined`; v5+ ignore any
+      // extra argument and always return a promise. Wrap a trailing callback for the
+      // legacy path and finish on the returned promise when there is one, so both APIs are
+      // covered without sniffing the version. Pre-v5 also validates arguments synchronously,
+      // so guard the call to finish the span instead of leaking it on a throw.
+      const lastIndex = args.length - 1
+      const callback = args[lastIndex]
+      if (typeof callback === 'function') {
+        args[lastIndex] = shimmer.wrapCallback(callback, callback => function (error) {
+          if (error) {
+            ctx.error = error
+            bulkWriteErrorCh.publish(ctx)
+          }
+          return bulkWriteFinishCh.runStores(ctx, callback, this, ...arguments)
+        })
+      }
+
+      let result
+      try {
+        result = bulkWrite.apply(this, args)
+      } catch (error) {
+        finishBulkWriteError(ctx, error)
+        throw error
+      }
+
+      if (result !== undefined && typeof result.then === 'function') {
+        result.then(function (value) {
+          ctx.result = value
+          bulkWriteFinishCh.publish(ctx)
+        }, function (error) {
+          finishBulkWriteError(ctx, error)
+        })
+      }
+
+      return result
+    })
+  }
+}
+
+/**
+ * @param {{ error?: unknown }} ctx
+ * @param {unknown} error
+ */
+function finishBulkWriteError (ctx, error) {
+  ctx.error = error
+  bulkWriteErrorCh.publish(ctx)
+  bulkWriteFinishCh.publish(ctx)
+}

package/packages/datadog-instrumentations/src/mongoose.js

@@ -6,6 +6,11 @@ const { wrapThen } = require('./helpers/promise')
 
 const startCh = channel('datadog:mongoose:model:filter:start')
 const finishCh = channel('datadog:mongoose:model:filter:finish')
+// Bound around the deferred query execution. The `bindStore` transform returns a
+// child store with the analysis marker set, covering the whole async scope that
+// reaches the mongodb driver. `runStores` enters the child only for that scope
+// and restores the parent on its own.
+const execCh = channel('datadog:mongoose:model:filter:exec')
 // this channel is for wrapping the callback of exec methods and handling store context
 const execStartCh = channel('apm:mongoose:exec:start')
 const execFinishCh = channel('apm:mongoose:exec:finish')
@@ -103,7 +108,13 @@ addHook({
         return startCh.runStores(ctx, () => {
           wrapCallbackIfExist(args, ctx)
 
-          const res = method.apply(this, args)
+          // A callback query executes synchronously inside `method.apply`, so the
+          // exec scope (and the deferred driver call it spawns) must be entered
+          // here. A promise query defers execution to a later `exec`/`then`, which
+          // is bound separately below.
+          const res = callbackWrapped
+            ? execCh.runStores(ctx, () => method.apply(this, args))
+            : method.apply(this, args)
 
           // if it is not callback, wrap exec method and its then
           if (!callbackWrapped) {
@@ -113,7 +124,7 @@ addHook({
                   wrapCallbackIfExist(args, ctx)
                 }
 
-                const execResult = originalExec.apply(this, args)
+                const execResult = execCh.runStores(ctx, () => originalExec.apply(this, args))
 
                 if (callbackWrapped || typeof execResult?.then !== 'function') {
                   return execResult

package/packages/datadog-instrumentations/src/playwright.js

@@ -840,6 +840,8 @@ function testEndHandler ({
     test._ddHasFailedAllRetries = true
   }
 
+  const willRetry = testWillRetry(test, testStatus)
+
   // this handles tests that do not go through the worker process (because they're skipped)
   if (shouldCreateTestSpan) {
     const testResult = results.at(-1)
@@ -850,7 +852,7 @@ function testEndHandler ({
       !test._ddIsEfdRetry
 
     const finalStatus = getFinalStatus({
-      isFinalExecution: !testWillRetry(test, testStatus),
+      isFinalExecution: !willRetry,
       isDisabled: test._ddIsDisabled,
       isQuarantined: test._ddIsQuarantined,
       isAtrRetry,
@@ -888,17 +890,19 @@ function testEndHandler ({
     }
   }
 
-  if (testSuiteToTestStatuses.has(testSuiteAbsolutePath)) {
-    testSuiteToTestStatuses.get(testSuiteAbsolutePath).push(testStatus)
-  } else {
-    testSuiteToTestStatuses.set(testSuiteAbsolutePath, [testStatus])
-  }
+  if (!willRetry) {
+    if (testSuiteToTestStatuses.has(testSuiteAbsolutePath)) {
+      testSuiteToTestStatuses.get(testSuiteAbsolutePath).push(testStatus)
+    } else {
+      testSuiteToTestStatuses.set(testSuiteAbsolutePath, [testStatus])
+    }
 
-  if (error) {
-    addErrorToTestSuite(testSuiteAbsolutePath, error)
+    if (error) {
+      addErrorToTestSuite(testSuiteAbsolutePath, error)
+    }
   }
 
-  if (!testWillRetry(test, testStatus)) {
+  if (!willRetry) {
     remainingTestsByFile[testSuiteAbsolutePath] = remainingTestsByFile[testSuiteAbsolutePath]
       .filter(currentTest => currentTest !== test)
   }