dd-trace 6.10.0 → 6.12.0

package/packages/datadog-instrumentations/src/playwright.js

@@ -45,6 +45,9 @@ const testSkipCh = channel('ci:playwright:test:skip')
 const testSessionStartCh = channel('ci:playwright:session:start')
 const testSessionConfigurationCh = channel('ci:playwright:session:configuration')
 const testSessionFinishCh = channel('ci:playwright:session:finish')
+const reporterErrorCh = channel('ci:playwright:reporter:error')
+const reporterRunSummaryCh = channel('ci:playwright:reporter:run-summary')
+const reporterSuiteHookErrorCh = channel('ci:playwright:reporter:suite-hook-error')
 
 const libraryConfigurationCh = channel('ci:playwright:library-configuration')
 const knownTestsCh = channel('ci:playwright:known-tests')
@@ -81,6 +84,8 @@ const isFailureScreenshotUploadEnabled =
   getValueFromEnvSources('DD_TEST_FAILURE_SCREENSHOTS_ENABLED') === true
 
 let applyRepeatEachIndex = null
+let reporterError
+let hasReporterError = false
 
 let startedSuites = []
 
@@ -108,7 +113,7 @@ let testManagementAttemptToFixRetries = 0
 let testManagementTests = {}
 let isImpactedTestsEnabled = false
 let modifiedFiles = {}
-let quarantinedButNotAttemptToFixFqns = new Set()
+let playwrightRunSummary
 let recordedTestOptimizationExecutions = new Set()
 let testsReportedInGenerateSummary = new Set()
 const newTestsWithDynamicNames = new Set()
@@ -134,8 +139,10 @@ const DD_PROPERTIES_REQUEST = 'ddPropertiesRequest'
 const DD_PROPERTIES_RESPONSE = 'ddProperties'
 const kDdPlaywrightDisabledTestIds = Symbol('ddPlaywrightDisabledTestIds')
 const kDdPlaywrightFailureScreenshots = Symbol('ddPlaywrightFailureScreenshots')
+const kDdPlaywrightReporterConfigured = Symbol('ddPlaywrightReporterConfigured')
 const kDdPlaywrightWorkerHostInstrumented = Symbol('ddPlaywrightWorkerHostInstrumented')
 const kDdPlaywrightWorkerInstrumented = Symbol('ddPlaywrightWorkerInstrumented')
+const instrumentedPlaywrightReporters = new WeakSet()
 const PLAYWRIGHT_FAILURE_SCREENSHOT_PATH_RE = /(?:^|[\\/])test-failed-\d+\.png$/
 const automaticFailureScreenshotPaths = new Set()
 
@@ -780,6 +787,7 @@ function finishTestSuiteIfDone (testSuiteAbsolutePath, projects) {
       testSourceLine: test.location.line,
       browserName,
       isNew: test._ddIsNew,
+      isAttemptToFix: test._ddIsAttemptToFix,
       isDisabled: test._ddIsDisabled,
       isModified: test._ddIsModified,
       isQuarantined: test._ddIsQuarantined,
@@ -827,6 +835,7 @@ function testEndHandler ({
 
     if (hookError) {
       addErrorToTestSuite(testSuiteAbsolutePath, hookError)
+      reporterSuiteHookErrorCh.publish(testSuiteAbsolutePath)
     }
     return
   }
@@ -912,9 +921,7 @@ function testEndHandler ({
 
   // Check if all EFD retries failed
   const efdRetryCount = getEfdRetryCountForTest(test)
-  if (efdRetryCount > 0 && testStatuses.length === efdRetryCount + 1 &&
-    (test._ddIsNew || test._ddIsModified) &&
-    isEarlyFlakeDetectionEnabled &&
+  if (isEfdManagedTest && efdRetryCount > 0 && testStatuses.length === efdRetryCount + 1 &&
     testStatuses.every(status => status === 'fail')) {
     test._ddHasFailedAllRetries = true
   }
@@ -1252,6 +1259,18 @@ function dispatcherHookNew (dispatcherExport, runWrapper) {
 function runAllTestsWrapper (runAllTests, playwrightVersion) {
   // Config parameter is only available from >=1.55.0
   return async function (config) {
+    reporterError = undefined
+    hasReporterError = false
+    playwrightRunSummary = undefined
+    let restoreReporterConsoleError
+    if (satisfies(playwrightVersion, '>=1.60.0') && config?.config) {
+      const DatadogPlaywrightReporter = require('./playwright-reporter')
+      restoreReporterConsoleError = DatadogPlaywrightReporter.restoreConsoleError
+      if (!config.config[kDdPlaywrightReporterConfigured]) {
+        Object.defineProperty(config.config, kDdPlaywrightReporterConfigured, { value: true })
+        config.config.reporter.unshift([require.resolve('./playwright-reporter')])
+      }
+    }
     rootDir = getRootDir(this, config)
     const projects = getProjectsFromRunner(this, config)
     const isFailureScreenshotEnabled = isFailureScreenshotCaptureEnabled(projects)
@@ -1367,7 +1386,27 @@ function runAllTestsWrapper (runAllTests, playwrightVersion) {
       }
     }
 
-    let runAllTestsReturn = await runAllTests.apply(this, arguments)
+    let runAllTestsReturn
+    try {
+      runAllTestsReturn = await runAllTests.apply(this, arguments)
+    } catch (error) {
+      try {
+        await getChannelPromise(testSessionFinishCh, {
+          status: 'fail',
+          error: hasReporterError ? reporterError : error,
+          isEarlyFlakeDetectionEnabled,
+          isEarlyFlakeDetectionFaulty,
+          isTestManagementTestsEnabled,
+        })
+      } catch (finalizationError) {
+        log.error('Playwright test session finalization error: %s', finalizationError)
+      }
+      reporterError = undefined
+      hasReporterError = false
+      throw error
+    } finally {
+      restoreReporterConsoleError?.()
+    }
 
     // Tests that have only skipped tests may reach this point
     // Skipped tests may or may not go through `testBegin` or `testEnd`
@@ -1397,23 +1436,8 @@ function runAllTestsWrapper (runAllTests, playwrightVersion) {
     const sessionStatus = runAllTestsReturn.status || runAllTestsReturn
 
     if (isTestManagementTestsEnabled && sessionStatus === 'failed') {
-      let totalFailedTestCount = 0
-      let totalPureQuarantinedFailedTestCount = 0
-
-      for (const [fqn, testStatuses] of testsToTestStatuses) {
-        // Only count as failed if the final status (after retries) is 'fail'
-        const lastStatus = testStatuses.at(-1)
-        if (lastStatus === 'fail') {
-          totalFailedTestCount += 1
-          if (quarantinedButNotAttemptToFixFqns.has(fqn)) {
-            totalPureQuarantinedFailedTestCount += 1
-          }
-        }
-      }
-
-      const totalIgnorableFailures = totalPureQuarantinedFailedTestCount
-
-      if (totalFailedTestCount > 0 && totalFailedTestCount === totalIgnorableFailures) {
+      const { failureCount, quarantinedFailureCount, hasIncompleteTests } = playwrightRunSummary || {}
+      if (!hasIncompleteTests && quarantinedFailureCount > 0 && quarantinedFailureCount === failureCount) {
         runAllTestsReturn = 'passed'
         preventedToFail = true
       }
@@ -1422,16 +1446,25 @@ function runAllTestsWrapper (runAllTests, playwrightVersion) {
     logTestOptimizationSummary({ attemptToFixExecutions, newTestsWithDynamicNames })
     loggedAttemptToFixTests.clear()
 
+    const finalizationError = reporterError
+    const finalStatus = hasReporterError
+      ? 'fail'
+      : (preventedToFail ? 'pass' : STATUS_TO_TEST_STATUS[sessionStatus])
     await getChannelPromise(testSessionFinishCh, {
-      status: preventedToFail ? 'pass' : STATUS_TO_TEST_STATUS[sessionStatus],
+      status: finalStatus,
+      error: finalizationError,
       isEarlyFlakeDetectionEnabled,
       isEarlyFlakeDetectionFaulty,
       isTestManagementTestsEnabled,
     })
+    if (hasReporterError) {
+      if (typeof runAllTestsReturn === 'string') runAllTestsReturn = 'failed'
+      else runAllTestsReturn.status = 'failed'
+    }
 
     startedSuites = []
     remainingTestsByFile = {}
-    quarantinedButNotAttemptToFixFqns = new Set()
+    playwrightRunSummary = undefined
     recordedTestOptimizationExecutions = new Set()
     testsReportedInGenerateSummary = new Set()
     efdManagedTestKeys.clear()
@@ -1444,6 +1477,8 @@ function runAllTestsWrapper (runAllTests, playwrightVersion) {
     ddPropertiesByTestId.clear()
     ddPropertiesRequestsByTestId.clear()
     disabledTestIds.clear()
+    reporterError = undefined
+    hasReporterError = false
 
     // TODO: we can trick playwright into thinking the session passed by returning
     // 'passed' here. We might be able to use this for both EFD and Test Management tests.
@@ -1451,6 +1486,70 @@ function runAllTestsWrapper (runAllTests, playwrightVersion) {
   }
 }
 
+/**
+ * Records errors thrown or rejected by Playwright reporters before Playwright converts them into run status.
+ *
+ * @param {object} reportersPackage
+ * @returns {object}
+ */
+function reportersHook (reportersPackage) {
+  return shimmer.wrap(reportersPackage, 'createReporters', createReporters => async function () {
+    const reporters = await createReporters.apply(this, arguments)
+    const DatadogPlaywrightReporter = require('./playwright-reporter')
+    reporters.unshift(new DatadogPlaywrightReporter({ captureReporterErrors: false }))
+    for (const reporter of reporters) {
+      if (instrumentedPlaywrightReporters.has(reporter)) continue
+
+      const onEnd = reporter.onEnd
+      const onExit = reporter.onExit
+      try {
+        if (typeof onEnd === 'function') {
+          reporter.onEnd = async function () {
+            try {
+              return await onEnd.apply(this, arguments)
+            } catch (error) {
+              recordReporterError(error)
+              throw error
+            }
+          }
+        }
+        if (typeof onExit === 'function') {
+          reporter.onExit = async function () {
+            try {
+              return await onExit.apply(this, arguments)
+            } catch (error) {
+              recordReporterError(error)
+              throw error
+            }
+          }
+        }
+      } catch {
+        continue
+      }
+      if (reporter.onEnd !== onEnd || reporter.onExit !== onExit) instrumentedPlaywrightReporters.add(reporter)
+    }
+    return reporters
+  }, { replaceGetter: true })
+}
+
+/**
+ * Records errors from the reporter multiplexer used by Playwright before 1.38.
+ *
+ * @param {object} reportersPackage
+ * @returns {object}
+ */
+function reporterMultiplexerHook (reportersPackage) {
+  shimmer.wrap(reportersPackage.Multiplexer.prototype, 'onEnd', onEnd => async function () {
+    try {
+      return await onEnd.apply(this, arguments)
+    } catch (error) {
+      recordReporterError(error)
+      throw error
+    }
+  })
+  return reportersPackage
+}
+
 function runnerHook (runnerExport, playwrightVersion) {
   shimmer.wrap(
     runnerExport.Runner.prototype,
@@ -1550,6 +1649,36 @@ pageGotoCh.subscribe({
   },
 })
 
+reporterErrorCh.subscribe((error) => {
+  recordReporterError(error)
+})
+
+reporterRunSummaryCh.subscribe((runSummary) => {
+  playwrightRunSummary = runSummary
+})
+
+/**
+ * Records a reporter failure even when the reporter throws a falsy value.
+ *
+ * @param {unknown} error
+ * @returns {void}
+ */
+function recordReporterError (error) {
+  if (hasReporterError) return
+
+  let normalizedError
+  try {
+    const errorObject = error && typeof error === 'object' ? error : undefined
+    normalizedError = new Error(errorObject?.message || errorObject?.value || String(error))
+    normalizedError.name = errorObject?.name || normalizedError.name
+    normalizedError.stack = errorObject?.stack || normalizedError.stack
+  } catch {
+    normalizedError = new Error('Playwright reporter failed')
+  }
+  reporterError = normalizedError
+  hasReporterError = true
+}
+
 /**
  * Records a path created by Playwright's automatic screenshot recorder.
  *
@@ -1597,6 +1726,12 @@ if (DD_MAJOR < 6) { // <1.38.0 is only supported up to version 5
     file: 'lib/runner/runner.js',
     versions: ['>=1.31.0 <1.38.0'],
   }, runnerHook)
+
+  addHook({
+    name: '@playwright/test',
+    file: 'lib/reporters/multiplexer.js',
+    versions: ['>=1.18.0 <1.38.0'],
+  }, reporterMultiplexerHook)
 }
 
 addHook({
@@ -1617,6 +1752,12 @@ addHook({
   versions: ['>=1.38.0 <1.60.0'],
 }, runnerHook)
 
+addHook({
+  name: 'playwright',
+  file: 'lib/runner/reporters.js',
+  versions: ['>=1.38.0 <1.60.0'],
+}, reportersHook)
+
 addHook({
   name: 'playwright',
   file: 'lib/runner/testRunner.js',
@@ -1747,11 +1888,6 @@ function processRootSuite (createRootSuiteReturnValue) {
       }
       if (testProperties.quarantined) {
         test._ddIsQuarantined = true
-        if (!testProperties.attemptToFix) {
-          // Do not skip quarantined tests, let them run and overwrite results post-run if they fail
-          const testFqn = getTestFullyQualifiedName(test)
-          quarantinedButNotAttemptToFixFqns.add(testFqn)
-        }
       }
       if (testProperties.attemptToFix) {
         test._ddIsAttemptToFix = true
@@ -1791,7 +1927,10 @@ function processRootSuite (createRootSuiteReturnValue) {
     const fileSuitesWithImpactedTestsToProjects = new Map()
     for (const impactedTest of impactedTests) {
       impactedTest._ddIsModified = true
-      if (shouldRunEarlyFlakeDetection() && impactedTest.expectedStatus !== 'skipped') {
+      if (shouldRunEarlyFlakeDetection() && impactedTest.expectedStatus !== 'skipped' &&
+        !impactedTest._ddIsAttemptToFix) {
+        // Prevent ATR or `--retries` from retrying tests that EFD manages.
+        impactedTest.retries = 0
         markEfdManagedTest(impactedTest)
         const fileSuite = getSuiteType(impactedTest, 'file')
         if (!fileSuitesWithImpactedTestsToProjects.has(fileSuite)) {
@@ -1799,14 +1938,15 @@ function processRootSuite (createRootSuiteReturnValue) {
         }
       }
     }
-    // If something change in the file, all tests in the file are impacted, hence the () => true filter
+    // If something changes in the file, all tests in the file are impacted, except attempt-to-fix tests managed above.
     applyRetriesToTests(
       fileSuitesWithImpactedTestsToProjects,
-      () => true,
+      (test) => !test._ddIsAttemptToFix,
       [
         '_ddIsModified',
         '_ddIsEfdRetry',
         (test) => (isKnownTestsEnabled && isNewTest(test) ? '_ddIsNew' : null),
+        (test) => test._ddIsQuarantined && '_ddIsQuarantined',
       ],
       earlyFlakeDetectionRetryPolicy.schedulingRetryCount,
       (copiedTest, originalTest, retryIndex) => {
@@ -1834,7 +1974,8 @@ function processRootSuite (createRootSuiteReturnValue) {
       const fileSuitesWithNewTestsToProjects = new Map()
       for (const newTest of newTests) {
         newTest._ddIsNew = true
-        if (shouldRunEarlyFlakeDetection() && newTest.expectedStatus !== 'skipped' && !newTest._ddIsModified) {
+        if (shouldRunEarlyFlakeDetection() && newTest.expectedStatus !== 'skipped' &&
+          !newTest._ddIsModified && !newTest._ddIsAttemptToFix) {
           // Prevent ATR or `--retries` from retrying tests that EFD manages.
           newTest.retries = 0
           markEfdManagedTest(newTest)
@@ -1847,8 +1988,12 @@ function processRootSuite (createRootSuiteReturnValue) {
 
       applyRetriesToTests(
         fileSuitesWithNewTestsToProjects,
-        isNewTest,
-        ['_ddIsNew', '_ddIsEfdRetry'],
+        (test) => !test._ddIsAttemptToFix && isNewTest(test),
+        [
+          '_ddIsNew',
+          '_ddIsEfdRetry',
+          (test) => test._ddIsQuarantined && '_ddIsQuarantined',
+        ],
         earlyFlakeDetectionRetryPolicy.schedulingRetryCount,
         (copiedTest, originalTest, retryIndex) => {
           markEfdRetryTest(copiedTest, retryIndex, originalTest)
@@ -2300,6 +2445,7 @@ function generateSummaryWrapper (generateSummary) {
             line: testSourceLine,
           },
           _ddIsNew: isNew,
+          _ddIsAttemptToFix: isAttemptToFix,
           _ddIsDisabled: isDisabled,
           _ddIsModified: isModified,
           _ddIsQuarantined: isQuarantined,
@@ -2314,6 +2460,7 @@ function generateSummaryWrapper (generateSummary) {
           testSourceLine,
           browserName,
           isNew,
+          isAttemptToFix,
           isDisabled,
           isModified,
           isQuarantined,

package/packages/datadog-instrumentations/src/router.js

@@ -596,6 +596,7 @@ const visitedParams = new WeakSet()
 function wrapHandleRequest (original) {
   return function wrappedHandleRequest (...args) {
     const req = args[0]
+    // eslint-disable-next-line no-restricted-syntax -- arbitrary param names; publishing {} sets a WAF address
     if (routerParamStartCh.hasSubscribers && !visitedParams.has(req.params) && Object.keys(req.params).length) {
       visitedParams.add(req.params)
 
@@ -635,6 +636,7 @@ function wrapParam (original) {
     args[1] = shimmer.wrapFunction(args[1], (originalFn) => {
       return function wrappedFn (...fnArgs) {
         const req = fnArgs[0]
+        // eslint-disable-next-line no-restricted-syntax -- arbitrary param names; publishing {} sets a WAF address
         if (routerParamStartCh.hasSubscribers && Object.keys(req.params).length && !visitedParams.has(req.params)) {
           visitedParams.add(req.params)
 

package/packages/datadog-instrumentations/src/vitest-main-no-worker-init.js

@@ -1058,7 +1058,7 @@ function getRepeatedTestReport (task, testName, testSuiteAbsolutePath, testPrope
   const errors = browserEnvironment
     ? normalizeVitestBrowserErrors(result?.errors)
     : result?.errors || []
-  const finalAttemptStatus = finalStatus(statuses)
+  const finalAttemptStatus = finalStatus(statuses, testProperties)
   const hasFailure = finalAttemptStatus === 'fail'
   const attempts = []
   const attemptCount = getRepeatedAttemptCount(task, statuses)
@@ -1095,8 +1095,7 @@ function getRepeatedTestReport (task, testName, testSuiteAbsolutePath, testPrope
         testProperties,
         type,
         statuses,
-        isFinalAttempt,
-        hasFailure
+        isFinalAttempt
       ),
       isRetry: index > 0,
       status: attemptStatus,
@@ -1136,11 +1135,10 @@ function getRepeatedTestReport (task, testName, testSuiteAbsolutePath, testPrope
  * @param {'attempt_to_fix'|'early_flake_detection'|'external'} type
  * @param {string[]} statuses
  * @param {boolean} isFinalAttempt
- * @param {boolean} hasFailure
  * @returns {boolean}
  */
-function hasFailedAllManagedRetries (task, testProperties, type, statuses, isFinalAttempt, hasFailure) {
-  if (!isFinalAttempt || !hasFailure || statuses.length === 0 || !statuses.every(status => status === 'fail')) {
+function hasFailedAllManagedRetries (task, testProperties, type, statuses, isFinalAttempt) {
+  if (!isFinalAttempt || statuses.length === 0 || !statuses.every(status => status === 'fail')) {
     return false
   }
 
@@ -1165,7 +1163,8 @@ function getAttemptToFixFinalStatus (statuses) {
   return statuses.includes('fail') ? 'fail' : 'pass'
 }
 
-function getEarlyFlakeDetectionFinalStatus (statuses) {
+function getEarlyFlakeDetectionFinalStatus (statuses, testProperties) {
+  if (testProperties.isDisabled || testProperties.isQuarantined) return 'skip'
   return statuses.includes('pass') ? 'pass' : 'fail'
 }
 
@@ -1274,7 +1273,6 @@ function reportFinalTestAttempt (testReport) {
         testProperties,
         'external',
         ['fail'],
-        true,
         true
       ),
       isRetry: (result?.retryCount || 0) > 0 || (result?.repeatCount || 0) > 0,

package/packages/datadog-instrumentations/src/vitest-main.js

@@ -67,6 +67,7 @@ const coverageWrappedProviders = new WeakSet()
 const finishWrappedContexts = new WeakSet()
 const runFilesWrappedPrototypes = new WeakSet()
 const activeRunFilesContexts = new WeakSet()
+const runErrorsByContext = new WeakMap()
 let isFlakyTestRetriesEnabled = false
 let flakyTestRetriesCount = 0
 let isEarlyFlakeDetectionEnabled = false
@@ -1195,15 +1196,18 @@ function getFinishWrapper (exitOrClose) {
     }
 
     const failedSuites = this.state.getFailedFilepaths()
-    let error
-    if (failedSuites.length) {
+    const runError = runErrorsByContext.get(this)
+    runErrorsByContext.delete(this)
+    let error = runError
+    if (!error && failedSuites.length) {
       error = new Error(`Test suites failed: ${failedSuites.length}.`)
     }
 
     const flushPromise = getChannelPromise(testSessionFinishCh, {
-      status: areAllSuitesSkipped ? 'skip' : getSessionStatus(this.state),
+      status: runError ? 'fail' : (areAllSuitesSkipped ? 'skip' : getSessionStatus(this.state)),
       testCodeCoverageLinesTotal,
       error,
+      isTestSessionFinalizationError: Boolean(runError),
       isEarlyFlakeDetectionEnabled,
       isEarlyFlakeDetectionFaulty,
       isTestManagementTestsEnabled,
@@ -1376,6 +1380,9 @@ function wrapVitestRunFiles (Vitest, frameworkVersion) {
     activeRunFilesContexts.add(this)
     try {
       return await runFiles.apply(this, arguments)
+    } catch (error) {
+      runErrorsByContext.set(this, error)
+      throw error
     } finally {
       activeRunFilesContexts.delete(this)
     }
@@ -1677,6 +1684,7 @@ async function reportTypecheckFile (file, sessionConfiguration, frameworkVersion
   }
 
   await getChannelPromise(testSuiteFinishCh, {
+    deferFlush: true,
     frameworkVersion,
     status: getTypecheckTaskStatus(file),
     ...testSuiteCtx.currentStore,

package/packages/datadog-plugin-aws-sdk/src/base.js

@@ -343,6 +343,7 @@ function getHostname (store, region) {
   if (!region) return
   const { awsParams, awsService } = store
   switch (awsService) {
+    case 'CloudWatchEvents':
     case 'EventBridge':
       return `events.${region}.amazonaws.com`
     case 'SQS':