dd-trace 5.86.0 → 5.88.0

package/packages/datadog-instrumentations/src/jest.js

@@ -21,6 +21,7 @@ const {
   getFormattedJestTestParameters,
   getJestTestName,
   getJestSuitesToRun,
+  getEfdRetryCount,
 } = require('../../datadog-plugin-jest/src/util')
 const { addHook, channel } = require('./helpers/instrument')
 
@@ -76,6 +77,7 @@ let hasUnskippableSuites = false
 let hasForcedToRunSuites = false
 let isEarlyFlakeDetectionEnabled = false
 let earlyFlakeDetectionNumRetries = 0
+let earlyFlakeDetectionSlowTestRetries = {}
 let earlyFlakeDetectionFaultyThreshold = 30
 let isEarlyFlakeDetectionFaulty = false
 let hasFilteredSkippableSuites = false
@@ -95,6 +97,12 @@ const attemptToFixRetriedTestsStatuses = new Map()
 const wrappedWorkers = new WeakSet()
 const testSuiteMockedFiles = new Map()
 const testsToBeRetried = new Set()
+// Per-test: how many EFD retries were determined after the first execution.
+const efdDeterminedRetries = new Map()
+// Tests whose first run exceeded the 5-min threshold — tagged "slow".
+const efdSlowAbortedTests = new Set()
+// Tests added as EFD new-test candidates (not ATF, not impacted).
+const efdNewTestCandidates = new Set()
 const testSuiteAbsolutePathsWithFastCheck = new Set()
 const testSuiteJestObjects = new Map()
 
@@ -129,6 +137,8 @@ function getTestEnvironmentOptions (config) {
   return {}
 }
 
+const MAX_IGNORED_TEST_NAMES = 10
+
 function getTestStats (testStatuses) {
   return testStatuses.reduce((acc, testStatus) => {
     acc[testStatus]++
@@ -136,6 +146,32 @@ function getTestStats (testStatuses) {
   }, { pass: 0, fail: 0 })
 }
 
+/**
+ * @param {string[]} efdNames
+ * @param {string[]} quarantineNames
+ * @param {number} totalCount
+ */
+function logIgnoredFailuresSummary (efdNames, quarantineNames, totalCount) {
+  const names = []
+  for (const n of efdNames) {
+    names.push({ name: n, reason: 'Early Flake Detection' })
+  }
+  for (const n of quarantineNames) {
+    names.push({ name: n, reason: 'Quarantine' })
+  }
+  const shown = names.slice(0, MAX_IGNORED_TEST_NAMES)
+  const more = names.length - shown.length
+  const moreSuffix = more > 0 ? `\n  ... and ${more} more` : ''
+  const list = shown.map(({ name, reason }) => `  • ${name} (${reason})`).join('\n')
+  const line = '-'.repeat(50)
+  // eslint-disable-next-line no-console -- Intentional user-facing message when exit code is flipped
+  console.warn(
+    `\n${line}\nDatadog Test Optimization\n${line}\n` +
+    `${totalCount} test failure(s) were ignored. Exit code set to 0.\n\n` +
+    `${list}${moreSuffix}\n`
+  )
+}
+
 function getWrappedEnvironment (BaseEnvironment, jestVersion) {
   return class DatadogEnvironment extends BaseEnvironment {
     constructor (config, context) {
@@ -169,7 +205,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
       this.isImpactedTestsEnabled = this.testEnvironmentOptions._ddIsImpactedTestsEnabled
 
       if (this.isKnownTestsEnabled) {
-        earlyFlakeDetectionNumRetries = this.testEnvironmentOptions._ddEarlyFlakeDetectionNumRetries
+        earlyFlakeDetectionSlowTestRetries = this.testEnvironmentOptions._ddEarlyFlakeDetectionSlowTestRetries ?? {}
         try {
           this.knownTestsForThisSuite = this.getKnownTestsForSuite(this.testEnvironmentOptions._ddKnownTests)
 
@@ -438,7 +474,13 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
         testContexts.set(event.test, ctx)
 
         testStartCh.runStores(ctx, () => {
-          for (const hook of event.test.parent.hooks) {
+          let p = event.test.parent
+          const hooks = []
+          while (p != null) {
+            hooks.push(...p.hooks)
+            p = p.parent
+          }
+          for (const hook of hooks) {
             let hookFn = hook.fn
             if (originalHookFns.has(hook)) {
               hookFn = originalHookFns.get(hook)
@@ -509,11 +551,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
             retriedTestsToNumAttempts.set(testFullName, 0)
             if (this.isEarlyFlakeDetectionEnabled) {
               testsToBeRetried.add(testFullName)
-              this.retryTest({
-                jestEvent: event,
-                retryCount: earlyFlakeDetectionNumRetries,
-                retryType: 'Early flake detection',
-              })
+              efdNewTestCandidates.add(testFullName)
+              // Cloning is deferred to test_done after the first execution,
+              // when we know the duration and can choose the right retry count.
             }
           }
         }
@@ -538,8 +578,8 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
         let attemptToFixFailed = false
         let failedAllTests = false
         let isAttemptToFix = false
+        const testName = getJestTestName(event.test, this.getShouldStripSeedFromTestName())
         if (this.isTestManagementTestsEnabled) {
-          const testName = getJestTestName(event.test, this.getShouldStripSeedFromTestName())
           isAttemptToFix = this.testManagementTestsForThisSuite?.attemptToFix?.includes(testName)
           if (isAttemptToFix) {
             if (attemptToFixRetriedTestsStatuses.has(testName)) {
@@ -564,9 +604,53 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
           }
         }
 
+        // EFD dynamic cloning: on first execution of a new EFD candidate,
+        // determine the retry count from the test's duration.
+        if (
+          this.isEarlyFlakeDetectionEnabled &&
+          this.isKnownTestsEnabled &&
+          efdNewTestCandidates.has(testName) &&
+          event.test.invocations === 1 &&
+          !efdDeterminedRetries.has(testName)
+        ) {
+          const durationMs = event.test.duration ?? 0
+          const retryCount = getEfdRetryCount(durationMs, earlyFlakeDetectionSlowTestRetries)
+          efdDeterminedRetries.set(testName, retryCount)
+          if (retryCount > 0) {
+            // Temporarily adjust jest-circus state so that retry tests are registered
+            // into the correct describe block and bypass the "tests have started" guard.
+            //
+            // Problem 1 (jest-circus ≤24): currentDescribeBlock points to ROOT during
+            // execution, and ROOT's tests loop already finished before children ran.
+            //
+            // Problem 2 (jest-circus ≥27): `hasStarted = true` causes `test()` to throw
+            // "Cannot add a test after tests have started running".
+            //
+            // Fix: temporarily point currentDescribeBlock to the test's parent (so retries
+            // land in the still-iterating children array) and set hasStarted = false (so the
+            // guard is bypassed). Both are restored immediately after scheduling the retries.
+            const originalDescribeBlock = state.currentDescribeBlock
+            const originalHasStarted = state.hasStarted
+            state.currentDescribeBlock = event.test.parent ?? originalDescribeBlock
+            state.hasStarted = false
+            this.retryTest({
+              jestEvent: {
+                testName: event.test.name,
+                fn: event.test.fn,
+                timeout: event.test.timeout,
+              },
+              retryCount,
+              retryType: 'Early flake detection',
+            })
+            state.currentDescribeBlock = originalDescribeBlock
+            state.hasStarted = originalHasStarted
+          } else {
+            efdSlowAbortedTests.add(testName)
+          }
+        }
+
         let isEfdRetry = false
         // We'll store the test statuses of the retries
-        const testName = getJestTestName(event.test, this.getShouldStripSeedFromTestName())
         if (this.isKnownTestsEnabled) {
           const isNewTest = retriedTestsToNumAttempts.has(testName)
           if (isNewTest) {
@@ -576,6 +660,14 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
             } else {
               newTestsTestStatuses.set(testName, [status])
             }
+            const testStatuses = newTestsTestStatuses.get(testName)
+            // Check if this is the last EFD retry.
+            // If it is, we'll set the failedAllTests flag to true if all the tests failed
+            const efdRetryCount = efdDeterminedRetries.get(testName) ?? 0
+            if (efdRetryCount > 0 && testStatuses.length === efdRetryCount + 1 &&
+              testStatuses.every(status => status === 'fail')) {
+              failedAllTests = true
+            }
           }
         }
 
@@ -630,6 +722,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
           attemptToFixFailed,
           isAtrRetry,
           finalStatus,
+          earlyFlakeAbortReason: efdSlowAbortedTests.has(testName) ? 'slow' : undefined,
         })
 
         if (promises.isProbeReady) {
@@ -641,6 +734,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
           test.errors = errors
         }
         atrSuppressedErrors.clear()
+        efdDeterminedRetries.clear()
+        efdSlowAbortedTests.clear()
+        efdNewTestCandidates.clear()
       }
       if (event.name === 'test_skip' || event.name === 'test_todo') {
         const testName = getJestTestName(event.test, this.getShouldStripSeedFromTestName())
@@ -661,7 +757,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
     getEfdResult ({ testName, isNewTest, isModifiedTest, isEfdRetry, numberOfExecutedRetries }) {
       const isEfdEnabled = this.isEarlyFlakeDetectionEnabled
       const isEfdActive = isEfdEnabled && (isNewTest || isModifiedTest)
-      const isLastEfdRetry = isEfdRetry && numberOfExecutedRetries >= (earlyFlakeDetectionNumRetries + 1)
+      const retryCount = efdDeterminedRetries.get(testName) ?? 0
+      const isSlowAbort = efdSlowAbortedTests.has(testName)
+      const isLastEfdRetry = (isEfdRetry && numberOfExecutedRetries >= (retryCount + 1)) || isSlowAbort
       const isFinalEfdTestExecution = isEfdActive && isLastEfdRetry
 
       let finalStatus
@@ -898,6 +996,7 @@ function getCliWrapper (isNewJestVersion) {
           isSuitesSkippingEnabled = libraryConfig.isSuitesSkippingEnabled
           isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled
           earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries
+          earlyFlakeDetectionSlowTestRetries = libraryConfig.earlyFlakeDetectionSlowTestRetries ?? {}
           earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold
           isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled
           isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled
@@ -994,11 +1093,18 @@ function getCliWrapper (isNewJestVersion) {
           coverageMap,
           numFailedTestSuites,
           numFailedTests,
+          numRuntimeErrorTestSuites = 0,
           numTotalTests,
           numTotalTestSuites,
+          runExecError,
+          wasInterrupted,
         },
       } = result
 
+      const hasSuiteLevelFailures = numRuntimeErrorTestSuites > 0
+      const hasRunLevelFailure = runExecError != null || wasInterrupted === true
+      const mustNotFlipSuccess = hasSuiteLevelFailures || hasRunLevelFailure
+
       let testCodeCoverageLinesTotal
 
       if (isUserCodeCoverageEnabled) {
@@ -1018,16 +1124,44 @@ function getCliWrapper (isNewJestVersion) {
        * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too.
        */
       let numEfdFailedTestsToIgnore = 0
+      const efdIgnoredNames = []
+      const quarantineIgnoredNames = []
+
+      // Build fullName -> suite map from results (for EFD display)
+      const fullNameToSuite = new Map()
+      for (const { testResults, testFilePath } of result.results.testResults) {
+        const suite = getTestSuitePath(testFilePath, result.globalConfig.rootDir)
+        for (const { fullName } of testResults) {
+          const name = testSuiteAbsolutePathsWithFastCheck.has(testFilePath)
+            ? fullName.replace(SEED_SUFFIX_RE, '')
+            : fullName
+          fullNameToSuite.set(name, suite)
+        }
+      }
+
+      /** @type {{ efdNames: string[], quarantineNames: string[], totalCount: number } | undefined} */
+      let ignoredFailuresSummary
       if (isEarlyFlakeDetectionEnabled) {
-        for (const testStatuses of newTestsTestStatuses.values()) {
+        for (const [testName, testStatuses] of newTestsTestStatuses) {
           const { pass, fail } = getTestStats(testStatuses)
           if (pass > 0) { // as long as one passes, we'll consider the test passed
             numEfdFailedTestsToIgnore += fail
+            const suite = fullNameToSuite.get(testName)
+            efdIgnoredNames.push(suite ? `${suite} › ${testName}` : testName)
           }
         }
         // If every test that failed was an EFD retry, we'll consider the suite passed
-        if (numEfdFailedTestsToIgnore !== 0 && result.results.numFailedTests === numEfdFailedTestsToIgnore) {
+        if (
+          !mustNotFlipSuccess &&
+          numEfdFailedTestsToIgnore !== 0 &&
+          result.results.numFailedTests === numEfdFailedTestsToIgnore
+        ) {
           result.results.success = true
+          ignoredFailuresSummary = {
+            efdNames: efdIgnoredNames,
+            quarantineNames: [],
+            totalCount: numEfdFailedTestsToIgnore,
+          }
         }
       }
 
@@ -1062,8 +1196,10 @@ function getCliWrapper (isNewJestVersion) {
           // This uses `attempt_to_fix` because this is always the main process and it's not formatted in camelCase
           if (testManagementTest?.attempt_to_fix && (testManagementTest?.quarantined || testManagementTest?.disabled)) {
             numFailedQuarantinedOrDisabledAttemptedToFixTests++
+            quarantineIgnoredNames.push(`${testSuite} › ${testName}`)
           } else if (testManagementTest?.quarantined) {
             numFailedQuarantinedTests++
+            quarantineIgnoredNames.push(`${testSuite} › ${testName}`)
           }
         }
 
@@ -1071,22 +1207,42 @@ function getCliWrapper (isNewJestVersion) {
         // Note that if a test is attempted to fix,
         // it's considered quarantined both if it's disabled and if it's quarantined
         // (it'll run but its status is ignored)
+        // Skip if EFD block already flipped (to avoid logging twice)
         if (
+          !result.results.success &&
+          !mustNotFlipSuccess &&
           (numFailedQuarantinedOrDisabledAttemptedToFixTests !== 0 || numFailedQuarantinedTests !== 0) &&
           result.results.numFailedTests ===
             numFailedQuarantinedTests + numFailedQuarantinedOrDisabledAttemptedToFixTests
         ) {
           result.results.success = true
+          ignoredFailuresSummary = {
+            efdNames: [],
+            quarantineNames: quarantineIgnoredNames,
+            totalCount: numFailedQuarantinedTests + numFailedQuarantinedOrDisabledAttemptedToFixTests,
+          }
         }
       }
 
       // Combined check: if all failed tests are accounted for by EFD (flaky retries) and/or quarantine,
       // we should consider the suite passed even when neither check alone covers all failures.
-      if (!result.results.success && (isEarlyFlakeDetectionEnabled || isTestManagementTestsEnabled)) {
+      if (
+        !result.results.success &&
+        !mustNotFlipSuccess &&
+        (isEarlyFlakeDetectionEnabled || isTestManagementTestsEnabled)
+      ) {
         const totalIgnoredFailures =
           numEfdFailedTestsToIgnore + numFailedQuarantinedTests + numFailedQuarantinedOrDisabledAttemptedToFixTests
-        if (totalIgnoredFailures !== 0 && result.results.numFailedTests === totalIgnoredFailures) {
+        if (
+          totalIgnoredFailures !== 0 &&
+          result.results.numFailedTests === totalIgnoredFailures
+        ) {
           result.results.success = true
+          ignoredFailuresSummary = {
+            efdNames: efdIgnoredNames,
+            quarantineNames: quarantineIgnoredNames,
+            totalCount: totalIgnoredFailures,
+          }
         }
       }
 
@@ -1144,6 +1300,14 @@ function getCliWrapper (isNewJestVersion) {
         })
       }
 
+      if (ignoredFailuresSummary) {
+        logIgnoredFailuresSummary(
+          ignoredFailuresSummary.efdNames,
+          ignoredFailuresSummary.quarantineNames,
+          ignoredFailuresSummary.totalCount
+        )
+      }
+
       numSkippedSuites = 0
 
       return result
@@ -1408,7 +1572,7 @@ addHook({
       _ddItrCorrelationId,
       _ddKnownTests,
       _ddIsEarlyFlakeDetectionEnabled,
-      _ddEarlyFlakeDetectionNumRetries,
+      _ddEarlyFlakeDetectionSlowTestRetries,
       _ddRepositoryRoot,
       _ddIsFlakyTestRetriesEnabled,
       _ddFlakyTestRetriesCount,

package/packages/datadog-instrumentations/src/kafkajs.js

@@ -24,22 +24,6 @@ const batchConsumerErrorCh = channel('apm:kafkajs:consume-batch:error')
 
 const disabledHeaderWeakSet = new WeakSet()
 
-function commitsFromEvent (event) {
-  const { payload: { groupId, topics } } = event
-  const commitList = []
-  for (const { topic, partitions } of topics) {
-    for (const { partition, offset } of partitions) {
-      commitList.push({
-        groupId,
-        partition,
-        offset,
-        topic,
-      })
-    }
-  }
-  consumerCommitCh.publish(commitList)
-}
-
 addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKafka) => {
   class Kafka extends BaseKafka {
     constructor (options) {
@@ -132,6 +116,7 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf
     }
 
     const kafkaClusterIdPromise = getKafkaClusterId(this)
+    let resolvedClusterId = null
 
     const eachMessageExtractor = (args, clusterId) => {
       const { topic, partition, message } = args[0]
@@ -146,13 +131,31 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf
 
     const consumer = createConsumer.apply(this, arguments)
 
-    consumer.on(consumer.events.COMMIT_OFFSETS, commitsFromEvent)
+    consumer.on(consumer.events.COMMIT_OFFSETS, (event) => {
+      const { payload: { groupId: commitGroupId, topics } } = event
+      const commitList = []
+      for (const { topic, partitions } of topics) {
+        for (const { partition, offset } of partitions) {
+          commitList.push({
+            groupId: commitGroupId,
+            partition,
+            offset,
+            topic,
+            clusterId: resolvedClusterId,
+          })
+        }
+      }
+      consumerCommitCh.publish(commitList)
+    })
 
     const run = consumer.run
     const groupId = arguments[0].groupId
 
     consumer.run = function ({ eachMessage, eachBatch, ...runArgs }) {
       const wrapConsume = (clusterId) => {
+        // In kafkajs COMMIT_OFFSETS always happens in the context of one synchronous run
+        // So this will always reference a correct cluster id
+        resolvedClusterId = clusterId
         return run({
           eachMessage: wrappedCallback(
             eachMessage,

package/packages/datadog-instrumentations/src/mocha/utils.js

@@ -271,6 +271,7 @@ function getOnTestEndHandler (config) {
     const testStatuses = testsStatuses.get(testName)
 
     const isLastAttempt = testStatuses.length === config.testManagementAttemptToFixRetries + 1
+    const isLastEfdRetry = testStatuses.length === config.earlyFlakeDetectionNumRetries + 1
 
     if (test._ddIsAttemptToFix && isLastAttempt) {
       if (testStatuses.includes('fail')) {
@@ -283,6 +284,11 @@ function getOnTestEndHandler (config) {
       }
     }
 
+    if (test._ddIsEfdRetry && isLastEfdRetry &&
+      testStatuses.every(status => status === 'fail')) {
+      hasFailedAllRetries = true
+    }
+
     const isAttemptToFixRetry = test._ddIsAttemptToFix && testStatuses.length > 1
     const isAtrRetry = config.isFlakyTestRetriesEnabled &&
       !test._ddIsAttemptToFix &&