dd-trace 6.7.0 → 6.8.0

package/packages/datadog-instrumentations/src/mocha/main.js

@@ -53,6 +53,7 @@ const {
   newTestsWithDynamicNames,
   attemptToFixExecutions,
   loggedAttemptToFixTests,
+  adjustRunnerFailuresForTestOptimization,
 } = require('./utils')
 
 require('./common')
@@ -114,17 +115,6 @@ function warnDeprecatedMochaVersion (frameworkVersion) {
   )
 }
 
-// Tests from workers do not come with `isFailed` method
-function isTestFailed (test) {
-  if (test.isFailed) {
-    return test.isFailed()
-  }
-  if (test.isPending) {
-    return !test.isPending() && test.state === 'failed'
-  }
-  return false
-}
-
 function getRootSuiteStatus (rootTests) {
   let status = 'pass'
   if (rootTests.every(t => t.isPending())) {
@@ -309,37 +299,7 @@ function getOnEndHandler (isParallel, onDone) {
       status = 'fail'
     }
 
-    if (config.isEarlyFlakeDetectionEnabled) {
-      /**
-       * If Early Flake Detection (EFD) is enabled the logic is as follows:
-       * - If all attempts for a test are failing, the test has failed and we will let the test process fail.
-       * - If just a single attempt passes, we will prevent the test process from failing.
-       * The rationale behind is the following: you may still be able to block your CI pipeline by gating
-       * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too.
-       */
-      for (const tests of Object.values(efdTests)) {
-        const failingEfdTests = tests.filter(test => isTestFailed(test))
-        const areAllEfdTestsFailing = failingEfdTests.length === tests.length
-        const nonQuarantinedFailingEfdTests = failingEfdTests.filter(test => !testsQuarantined.has(test))
-        if (nonQuarantinedFailingEfdTests.length && !areAllEfdTestsFailing) {
-          this.stats.failures -= nonQuarantinedFailingEfdTests.length
-          this.failures -= nonQuarantinedFailingEfdTests.length
-        }
-      }
-    }
-
-    // We subtract the errors from quarantined tests from the total number of failures.
-    // Attempt-to-fix tests ignore quarantine/disabled suppression and keep their framework result.
-    if (config.isTestManagementTestsEnabled) {
-      let numFailedQuarantinedTests = 0
-      for (const test of testsQuarantined) {
-        if (isTestFailed(test)) {
-          numFailedQuarantinedTests++
-        }
-      }
-      this.stats.failures -= numFailedQuarantinedTests
-      this.failures -= numFailedQuarantinedTests
-    }
+    adjustRunnerFailuresForTestOptimization(this, config)
 
     // Recompute status after EFD and quarantine adjustments have reduced failure counts
     if (status === 'fail') {

package/packages/datadog-instrumentations/src/mocha/utils.js

@@ -48,6 +48,59 @@ function waitForHitProbe () {
 }
 const loggedAttemptToFixTests = new Set()
 
+/**
+ * Checks whether a Mocha test failed, including serialized tests from parallel workers.
+ *
+ * @param {object} test
+ * @returns {boolean}
+ */
+function isTestFailed (test) {
+  if (test.isFailed) {
+    return test.isFailed()
+  }
+  if (test.isPending) {
+    return !test.isPending() && test.state === 'failed'
+  }
+  return false
+}
+
+/**
+ * Applies EFD and quarantine failure suppression to a completed Mocha runner.
+ *
+ * @param {object} runner
+ * @param {object} config
+ * @returns {void}
+ */
+function adjustRunnerFailuresForTestOptimization (runner, config) {
+  if (config.isEarlyFlakeDetectionEnabled) {
+    for (const tests of Object.values(efdTests)) {
+      const executedEfdTests = tests.filter(test => !test.isPending())
+      const failingEfdTests = tests.filter(test => isTestFailed(test))
+      const areAllEfdTestsFailing = failingEfdTests.length === executedEfdTests.length
+      const nonQuarantinedFailingEfdTests = failingEfdTests.filter(test => !testsQuarantined.has(test))
+      if (nonQuarantinedFailingEfdTests.length && !areAllEfdTestsFailing) {
+        if (runner.stats) {
+          runner.stats.failures -= nonQuarantinedFailingEfdTests.length
+        }
+        runner.failures -= nonQuarantinedFailingEfdTests.length
+      }
+    }
+  }
+
+  if (config.isTestManagementTestsEnabled) {
+    let numFailedQuarantinedTests = 0
+    for (const test of testsQuarantined) {
+      if (isTestFailed(test)) {
+        numFailedQuarantinedTests++
+      }
+    }
+    if (runner.stats) {
+      runner.stats.failures -= numFailedQuarantinedTests
+    }
+    runner.failures -= numFailedQuarantinedTests
+  }
+}
+
 function getAfterEachHooks (testOrHook) {
   const hooks = []
 
@@ -165,6 +218,7 @@ function isDatadogManagedRetryTest (test, config) {
  * Checks whether a runnable belongs to an Early Flake Detection execution.
  * @param {{
  *   _ddIsAttemptToFix?: boolean,
+ *   _ddIsDisabled?: boolean,
  *   _ddIsEfdRetry?: boolean,
  *   _ddIsModified?: boolean,
  *   _ddIsNew?: boolean
@@ -174,6 +228,7 @@ function isDatadogManagedRetryTest (test, config) {
  */
 function isEarlyFlakeDetectionTest (test, config) {
   return !test._ddIsAttemptToFix &&
+    !test._ddIsDisabled &&
     config.isEarlyFlakeDetectionEnabled &&
     (test._ddIsEfdRetry || test._ddIsNew || test._ddIsModified)
 }
@@ -949,7 +1004,12 @@ function getRunTestsWrapper (runTests, config) {
           onDone: (isModified) => {
             if (isModified) {
               test._ddIsModified = true
-              if (!test.isPending() && !test._ddIsAttemptToFix && config.isEarlyFlakeDetectionEnabled) {
+              if (
+                !test.isPending() &&
+                !test._ddIsDisabled &&
+                !test._ddIsAttemptToFix &&
+                config.isEarlyFlakeDetectionEnabled
+              ) {
                 retryTest(
                   test,
                   getConfiguredEfdRetryCount(config),
@@ -968,7 +1028,12 @@ function getRunTestsWrapper (runTests, config) {
       suite.tests.forEach((test) => {
         if (!test.isPending() && isNewTest(test, config.knownTests)) {
           test._ddIsNew = true
-          if (config.isEarlyFlakeDetectionEnabled && !test._ddIsAttemptToFix && !test._ddIsModified) {
+          if (
+            config.isEarlyFlakeDetectionEnabled &&
+            !test._ddIsDisabled &&
+            !test._ddIsAttemptToFix &&
+            !test._ddIsModified
+          ) {
             retryTest(
               test,
               getConfiguredEfdRetryCount(config),
@@ -1015,4 +1080,5 @@ module.exports = {
   testsStatuses,
   attemptToFixExecutions,
   loggedAttemptToFixTests,
+  adjustRunnerFailuresForTestOptimization,
 }

package/packages/datadog-instrumentations/src/mocha/webdriverio-protocol.js

@@ -0,0 +1,19 @@
+'use strict'
+
+const {
+  createWebdriverioWorkerMessage,
+  WEBDRIVERIO_WORKER_ENV,
+  WEBDRIVERIO_WORKER_EVENT,
+  WEBDRIVERIO_WORKER_ORIGIN,
+} = require('../../../dd-trace/src/ci-visibility/exporters/test-worker/webdriverio')
+
+module.exports = {
+  CONFIGURATION_REQUEST: 'dd:test-optimization:webdriverio:configuration:request',
+  CONFIGURATION_RESPONSE: 'dd:test-optimization:webdriverio:configuration:response',
+  createWebdriverioWorkerMessage,
+  SUITE_FINISH: 'dd:test-optimization:webdriverio:test-suite:finish',
+  WORKER_READY: 'dd:test-optimization:webdriverio:worker:ready',
+  WEBDRIVERIO_WORKER_ENV,
+  WEBDRIVERIO_WORKER_EVENT,
+  WEBDRIVERIO_WORKER_ORIGIN,
+}