dd-trace 6.10.0 → 6.12.0

package/packages/datadog-instrumentations/src/mocha/utils.js

@@ -35,9 +35,13 @@ const testSuiteErrorCh = channel('ci:mocha:test-suite:error')
 /** @typedef {{ length: number, [index: number]: unknown } & Iterable<unknown>} ArgumentsLike */
 
 const testToContext = new WeakMap()
+const originalEfdFns = new WeakMap()
 const originalFns = new WeakMap()
+const originalPendingByTest = new WeakMap()
+const originalRetriesByTest = new WeakMap()
 const testToStartLine = new WeakMap()
 const testFileToSuiteCtx = new Map()
+const datadogRetryOriginals = new WeakMap()
 const wrappedFunctions = new WeakSet()
 const newTests = {}
 const efdTests = {}
@@ -169,6 +173,7 @@ function wrapOriginalEfdTest (test, retryPolicy) {
   }
   test._ddEfdDurationWrapped = true
   const originalFn = test.fn
+  originalEfdFns.set(test, originalFn)
   test.fn = shimmer.wrapFunction(originalFn, originalFn => function () {
     const start = performance.now()
     const recordDuration = () => {
@@ -265,6 +270,7 @@ function isEarlyFlakeDetectionTest (test, config) {
 function retryTest (test, numRetries, tags, retryPolicy) {
   const suite = test.parent
   const isEfdRetry = tags.includes('_ddIsEfdRetry')
+  if (!originalRetriesByTest.has(test)) originalRetriesByTest.set(test, test.retries())
   disableMochaRetries(test)
   if (isEfdRetry) {
     wrapOriginalEfdTest(test, retryPolicy)
@@ -272,6 +278,7 @@ function retryTest (test, numRetries, tags, retryPolicy) {
   for (let retryIndex = 0; retryIndex < numRetries; retryIndex++) {
     const clonedTest = test.clone()
     disableMochaRetries(clonedTest)
+    datadogRetryOriginals.set(clonedTest, test)
     suite.addTest(clonedTest)
     if (isEfdRetry) {
       clonedTest._ddEfdRetryIndex = retryIndex + 1
@@ -292,6 +299,94 @@ function retryTest (test, numRetries, tags, retryPolicy) {
         clonedTest[tag] = true
       }
     }
+    if (clonedTest._ddIsQuarantined && !clonedTest._ddIsAttemptToFix) {
+      testsQuarantined.add(clonedTest)
+    }
+  }
+}
+
+/**
+ * Restores a runnable function wrapped with its Test Optimization context.
+ *
+ * @param {import('mocha').Runnable} runnable
+ * @returns {void}
+ */
+function restoreRunnableFunction (runnable) {
+  const wrappedFunction = runnable.fn
+  if (!wrappedFunctions.has(wrappedFunction)) return
+
+  runnable.fn = originalFns.get(wrappedFunction)
+}
+
+/**
+ * Restores a test function wrapped to measure its EFD duration.
+ *
+ * @param {import('mocha').Test} test
+ * @returns {void}
+ */
+function restoreEfdTestFunction (test) {
+  if (!originalEfdFns.has(test)) return
+
+  test.fn = originalEfdFns.get(test)
+  originalEfdFns.delete(test)
+  delete test._ddEfdDurationWrapped
+}
+
+/**
+ * Clears state that belongs to one Mocha runner execution and removes retry clones from prior executions.
+ *
+ * @param {import('mocha').Suite} rootSuite
+ * @returns {void}
+ */
+function resetRunState (rootSuite) {
+  for (const key of Object.keys(newTests)) delete newTests[key]
+  for (const key of Object.keys(efdTests)) delete efdTests[key]
+  newTestsWithDynamicNames.clear()
+  testsAttemptToFix.clear()
+  testsQuarantined.clear()
+  testsStatuses.clear()
+  efdRetryCountByTestFullName.clear()
+  efdSlowAbortedTests.clear()
+  attemptToFixExecutions.clear()
+  loggedAttemptToFixTests.clear()
+
+  const suites = [rootSuite]
+  while (suites.length) {
+    const suite = suites.pop()
+    const originalTests = []
+    const retainedTests = new Set()
+    for (const runnable of suite.tests) {
+      const test = datadogRetryOriginals.get(runnable) || runnable
+      if (retainedTests.has(test)) continue
+      retainedTests.add(test)
+
+      restoreRunnableFunction(test)
+      restoreEfdTestFunction(test)
+      if (originalPendingByTest.has(test)) {
+        test.pending = originalPendingByTest.get(test)
+        originalPendingByTest.delete(test)
+      }
+      if (originalRetriesByTest.has(test)) {
+        test.retries(originalRetriesByTest.get(test))
+        originalRetriesByTest.delete(test)
+      }
+
+      delete test._ddIsAttemptToFix
+      delete test._ddIsDisabled
+      delete test._ddIsQuarantined
+      delete test._ddIsModified
+      delete test._ddIsNew
+      delete test._ddShouldSkipEfdRetry
+      delete test._ddTestFinishStarted
+      delete test._ddTestFinishPublished
+      delete test._ddIsFinalAttempt
+      delete test._ddHookFailed
+      delete test._ddReporterStartFailed
+      delete test._ddReporterTerminalFailed
+      originalTests.push(test)
+    }
+    suite.tests = originalTests
+    suites.push(...suite.suites)
   }
 }
 
@@ -371,6 +466,19 @@ function getTestContext (test) {
   return testToContext.get(key)
 }
 
+/**
+ * Claims publication of a test finish event across Mocha's terminal event paths.
+ *
+ * @param {object} test
+ * @returns {boolean}
+ */
+function startTestFinish (test) {
+  if (test._ddTestFinishStarted || test._ddTestFinishPublished) return false
+
+  test._ddTestFinishStarted = true
+  return true
+}
+
 /**
  * Copies Test Management metadata from Mocha's original runnable to its native retry clone.
  * @param {{
@@ -413,11 +521,7 @@ function runnableWrapper (RunnablePackage, libraryConfig) {
     const test = isTestHook ? this.ctx.currentTest : this
 
     // we restore the original user defined function
-    if (wrappedFunctions.has(this.fn)) {
-      const originalFn = originalFns.get(this.fn)
-      this.fn = originalFn
-      wrappedFunctions.delete(this.fn)
-    }
+    restoreRunnableFunction(this)
 
     if (isDatadogManagedRetryTest(test, libraryConfig)) {
       disableMochaRetries(this)
@@ -460,11 +564,7 @@ function getOnTestHandler (isMain) {
 
     // This may be a retry. If this is the case, `test.fn` is already wrapped,
     // so we need to restore it.
-    if (wrappedFunctions.has(test.fn)) {
-      const originalFn = originalFns.get(test.fn)
-      test.fn = originalFn
-      wrappedFunctions.delete(test.fn)
-    }
+    restoreRunnableFunction(test)
 
     inheritDatadogPropertiesFromRetriedTest(test)
 
@@ -527,6 +627,9 @@ function getOnTestHandler (isMain) {
     }
 
     if (!isAttemptToFix && isDisabled) {
+      if (!originalPendingByTest.has(test)) {
+        originalPendingByTest.set(test, test.pending)
+      }
       test.pending = true
     }
 
@@ -701,9 +804,16 @@ function getOnTestEndHandler (config, finalAttemptHandlers) {
     if (test._ddShouldSkipEfdRetry) {
       return
     }
+    const shouldWaitForHitProbe = test._retriedTest?._ddShouldWaitForHitProbe
     const ctx = getTestContext(test)
     const status = getTestStatus(test)
-    const shouldFinishTest = ctx && (!getAfterEachHooks(test).length || (test._ddIsDisabled && !test._ddIsAttemptToFix))
+    const shouldFinishTest = ctx && (
+      test._ddReporterStartFailed ||
+      test._ddReporterTerminalFailed ||
+      !getAfterEachHooks(test).length ||
+      (test._ddIsDisabled && !test._ddIsAttemptToFix)
+    )
+    if (shouldFinishTest && !startTestFinish(test)) return
     let testFinishInfo
     let isFinalAttempt = false
 
@@ -729,11 +839,12 @@ function getOnTestEndHandler (config, finalAttemptHandlers) {
       finalAttemptHandlers?.onStart?.(test)
     }
 
-    if (test._retriedTest?._ddShouldWaitForHitProbe) {
+    if (shouldWaitForHitProbe) {
       await waitForHitProbe()
     }
 
     if (shouldFinishTest) {
+      test._ddTestFinishPublished = true
       testFinishCh.publish({
         status,
         hasBeenRetried: isMochaRetry(test),
@@ -760,10 +871,11 @@ function getOnHookEndHandler (config, finalAttemptHandlers) {
         const ctx = getTestContext(test)
         // Disabled tests are already finished in getOnTestEndHandler,
         // skip to avoid double-publishing
-        if (ctx && (!test._ddIsDisabled || test._ddIsAttemptToFix)) {
+        if (ctx && (!test._ddIsDisabled || test._ddIsAttemptToFix) && startTestFinish(test)) {
           const testFinishInfo = getTestFinishInfo(test, status, config, ctx.err || test.err)
           const isFinalAttempt = testFinishInfo.finalStatus !== undefined
           const publishTestFinish = () => {
+            test._ddTestFinishPublished = true
             testFinishCh.publish({
               status,
               hasBeenRetried: isMochaRetry(test),
@@ -887,11 +999,13 @@ function getOnFailHandler (isMain, config) {
       testContext = getTestContext(test)
     }
     if (testContext) {
-      if (isHook) {
-        err.message = `${testOrHook.fullTitle()}: ${err.message}`
-        testContext.err = err
+      if (isHook && startTestFinish(test)) {
+        const hookError = new Error(`${testOrHook.fullTitle()}: ${err.message}`, { cause: err })
+        hookError.name = err.name
+        hookError.stack = err.stack
+        testContext.err = hookError
         errorCh.runStores(testContext, () => {})
-        const testFinishInfo = getTestFinishInfo(test, 'fail', config, err)
+        const testFinishInfo = getTestFinishInfo(test, 'fail', config, hookError)
         // ATR never retries hook failures: this.retries(N) is set in runnableWrapper
         // which only runs when the test function executes — hooks bypass that path,
         // so _retries stays at -1 and getIsLastRetry returns false, leaving finalStatus
@@ -906,6 +1020,7 @@ function getOnFailHandler (isMain, config) {
         // test.state is never set to 'failed' for hook failures (Mocha marks the hook,
         // not the test). Flag it so finishRootSuiteForFile can compute the correct status.
         test._ddHookFailed = true
+        test._ddTestFinishPublished = true
         testFinishCh.publish({
           status: 'fail',
           hasBeenRetried: isMochaRetry(test),
@@ -1034,7 +1149,7 @@ function getRunTestsWrapper (runTests, config) {
                 retryTest(
                   test,
                   config.earlyFlakeDetectionRetryPolicy.schedulingRetryCount,
-                  ['_ddIsModified', '_ddIsEfdRetry'],
+                  ['_ddIsModified', '_ddIsEfdRetry', test._ddIsQuarantined && '_ddIsQuarantined'],
                   config.earlyFlakeDetectionRetryPolicy
                 )
               }
@@ -1053,7 +1168,7 @@ function getRunTestsWrapper (runTests, config) {
             retryTest(
               test,
               config.earlyFlakeDetectionRetryPolicy.schedulingRetryCount,
-              ['_ddIsNew', '_ddIsEfdRetry'],
+              ['_ddIsNew', '_ddIsEfdRetry', test._ddIsQuarantined && '_ddIsQuarantined'],
               config.earlyFlakeDetectionRetryPolicy
             )
           }
@@ -1088,6 +1203,7 @@ module.exports = {
   getOnPendingHandler,
   testFileToSuiteCtx,
   getRunTestsWrapper,
+  resetRunState,
   newTests,
   efdTests,
   newTestsWithDynamicNames,

package/packages/datadog-instrumentations/src/playwright-reporter.js

@@ -0,0 +1,247 @@
+'use strict'
+
+const { channel } = require('dc-polyfill')
+
+const reporterErrorCh = channel('ci:playwright:reporter:error')
+const reporterRunSummaryCh = channel('ci:playwright:reporter:run-summary')
+const reporterSuiteHookErrorCh = channel('ci:playwright:reporter:suite-hook-error')
+const suitesWithHookErrors = new Set()
+const PLAYWRIGHT_REPORTER_ERROR_MESSAGE = 'Error in reporter'
+const PLAYWRIGHT_REPORTER_ERROR_CALLER_RE =
+  /^\s*at wrapAsync .*?[\\/]playwright[\\/]lib[\\/]runner[\\/]index\.js:\d+:\d+\)?$/
+
+/**
+ * Identifies the private Playwright 1.60-1.61 reporter error fallback without
+ * interpreting identical user console output as a framework error.
+ *
+ * @param {unknown} message - First console.error argument
+ * @returns {boolean}
+ */
+function isPlaywrightReporterError (message) {
+  if (message !== PLAYWRIGHT_REPORTER_ERROR_MESSAGE) return false
+
+  const originalPrepareStackTrace = Error.prepareStackTrace
+  try {
+    Error.prepareStackTrace = (_, callSites) => callSites
+    const stack = new Error('Playwright reporter error provenance').stack
+    return PLAYWRIGHT_REPORTER_ERROR_CALLER_RE.test(`at ${stack?.[2]?.toString() || ''}`)
+  } finally {
+    Error.prepareStackTrace = originalPrepareStackTrace
+  }
+}
+
+/**
+ * Returns whether a finalized Playwright result contains a failed suite hook.
+ *
+ * @param {Array<object>} steps
+ * @returns {boolean}
+ */
+function hasFailedSuiteHook (steps) {
+  if (!steps) return false
+
+  for (const step of steps) {
+    if (step.error && (step.title === 'beforeAll hook' || step.title === 'afterAll hook')) return true
+    if (hasFailedSuiteHook(step.steps)) return true
+  }
+  return false
+}
+
+class DatadogPlaywrightReporter {
+  /**
+   * Creates a reporter that observes the finalized Playwright result independently of user reporters.
+   *
+   * @param {object} [options]
+   * @param {boolean} [options.captureReporterErrors]
+   */
+  constructor ({ captureReporterErrors = true } = {}) {
+    this.captureReporterErrors = captureReporterErrors
+    this.fatalErrorCount = 0
+  }
+
+  /**
+   * Identifies this reporter as implementing Playwright's reporter v2 contract.
+   *
+   * @returns {'v2'}
+   */
+  version () {
+    return 'v2'
+  }
+
+  /**
+   * Implements the reporter v2 lifecycle hook required by older Playwright versions.
+   *
+   * @param {object} config
+   * @returns {void}
+   */
+  onConfigure (config) {
+    this.failOnFlakyTests = config.failOnFlakyTests
+  }
+
+  /**
+   * Restores console error after Playwright completes the reporter lifecycle.
+   *
+   * @returns {void}
+   */
+  static restoreConsoleError () {
+    // eslint-disable-next-line no-console
+    if (console.error === DatadogPlaywrightReporter.consoleError) {
+      try {
+        // eslint-disable-next-line no-console
+        console.error = DatadogPlaywrightReporter.originalConsoleError
+      } catch {}
+    }
+    DatadogPlaywrightReporter.consoleError = undefined
+    DatadogPlaywrightReporter.originalConsoleError = undefined
+  }
+
+  /**
+   * Records the suite so finalized test outcomes can be counted in onEnd.
+   *
+   * @param {object} configOrSuite
+   * @param {object} [suite]
+   * @returns {void}
+   */
+  onBegin (configOrSuite, suite) {
+    this.suite = suite || configOrSuite
+    if (suite) this.failOnFlakyTests = configOrSuite.failOnFlakyTests
+    suitesWithHookErrors.clear()
+  }
+
+  /**
+   * Implements the reporter v2 lifecycle hook required by older Playwright versions.
+   *
+   * @returns {void}
+   */
+  onTestBegin () {}
+
+  /**
+   * Implements the reporter v2 lifecycle hook required by older Playwright versions.
+   *
+   * @returns {void}
+   */
+  onStdOut () {}
+
+  /**
+   * Implements the reporter v2 lifecycle hook required by older Playwright versions.
+   *
+   * @returns {void}
+   */
+  onStdErr () {}
+
+  /**
+   * Implements the reporter v2 lifecycle hook required by older Playwright versions.
+   *
+   * @returns {void}
+   */
+  onTestEnd () {}
+
+  /**
+   * Implements the reporter v2 lifecycle hook required by older Playwright versions.
+   *
+   * @returns {void}
+   */
+  onStepBegin () {}
+
+  /**
+   * Implements the reporter v2 lifecycle hook required by older Playwright versions.
+   *
+   * @returns {void}
+   */
+  onStepEnd () {}
+
+  /**
+   * Marks the beginning of reporter finalization so later reporter errors can be identified.
+   *
+   * @returns {void}
+   */
+  onEnd () {
+    let failureCount = this.fatalErrorCount
+    let quarantinedFailureCount = 0
+    let hasIncompleteTests = suitesWithHookErrors.size > 0
+    const tests = this.suite?.allTests?.()
+    if (tests) {
+      for (const test of tests) {
+        const outcome = test.outcome()
+        const finalResult = test.results.at(-1)
+        hasIncompleteTests ||= hasFailedSuiteHook(finalResult?.steps)
+        if (outcome === 'unexpected' || (outcome === 'flaky' && this.failOnFlakyTests)) {
+          failureCount += 1
+          const finalResultStatus = finalResult?.status
+          const hasFailed = finalResultStatus === 'failed' || finalResultStatus === 'timedOut'
+          if (outcome === 'unexpected' && hasFailed && test._ddIsQuarantined && !test._ddIsAttemptToFix) {
+            quarantinedFailureCount += 1
+          }
+        } else if (outcome === 'skipped' && !test._ddShouldSkipEfdRetry &&
+          (!test._ddIsDisabled || test._ddIsAttemptToFix)) {
+          const { results } = test
+          hasIncompleteTests ||= results.some(result => result.status === 'interrupted') ||
+            !results.length || test.expectedStatus !== 'skipped'
+        }
+      }
+    }
+    reporterRunSummaryCh.publish({ failureCount, quarantinedFailureCount, hasIncompleteTests })
+    suitesWithHookErrors.clear()
+
+    if (!this.captureReporterErrors) return
+
+    this.isFinalizing = true
+    // Playwright 1.60 and 1.61 only expose reporter errors through this exact console call.
+    // eslint-disable-next-line no-console
+    const originalConsoleError = console.error
+    const reporter = this
+    const consoleError = function (message, error) {
+      if (isPlaywrightReporterError(message)) {
+        reporter.onError(error)
+      }
+      return originalConsoleError.apply(this, arguments)
+    }
+    try {
+      // eslint-disable-next-line no-console
+      console.error = consoleError
+    } catch {
+      return
+    }
+    // An accessor may ignore the assignment without throwing.
+    // eslint-disable-next-line no-console
+    if (console.error !== consoleError) return
+
+    DatadogPlaywrightReporter.originalConsoleError = originalConsoleError
+    DatadogPlaywrightReporter.consoleError = consoleError
+  }
+
+  /**
+   * Implements the reporter v2 lifecycle hook required by older Playwright versions.
+   *
+   * @returns {void}
+   */
+  onExit () {}
+
+  /**
+   * Reports errors emitted by Playwright while later reporters are finalizing.
+   *
+   * @param {unknown} error
+   * @returns {void}
+   */
+  onError (error) {
+    if (this.isFinalizing) {
+      reporterErrorCh.publish(error)
+    } else {
+      this.fatalErrorCount += 1
+    }
+  }
+
+  /**
+   * Keeps the internal reporter from affecting Playwright's output reporter selection.
+   *
+   * @returns {boolean}
+   */
+  printsToStdio () {
+    return false
+  }
+}
+
+reporterSuiteHookErrorCh.subscribe((testSuiteAbsolutePath) => {
+  suitesWithHookErrors.add(testSuiteAbsolutePath)
+})
+
+module.exports = DatadogPlaywrightReporter