dd-trace 6.8.0 → 6.9.0

package/packages/datadog-instrumentations/src/vitest-util.js

@@ -1,6 +1,12 @@
 'use strict'
 
+const fs = require('node:fs')
+
 const log = require('../../dd-trace/src/log')
+const {
+  EMPTY_EFD_RETRY_POLICY,
+  hasEfdRetries,
+} = require('../../dd-trace/src/ci-visibility/efd-retry-policy')
 const { channel } = require('./helpers/instrument')
 
 // test hooks
@@ -28,9 +34,25 @@ const testManagementTestsCh = channel('ci:vitest:test-management-tests')
 const modifiedFilesCh = channel('ci:vitest:modified-files')
 
 const workerReportTraceCh = channel('ci:vitest:worker-report:trace')
+const workerReportCoverageCh = channel('ci:vitest:worker-report:coverage')
 const workerReportLogsCh = channel('ci:vitest:worker-report:logs')
+const workerReportTelemetryCh = channel('ci:vitest:worker-report:telemetry')
 const codeCoverageReportCh = channel('ci:vitest:coverage-report')
 
+/**
+ * Resolves a path without failing Test Optimization when the path is unavailable.
+ *
+ * @param {string} filepath
+ * @returns {string}
+ */
+function realpath (filepath) {
+  try {
+    return fs.realpathSync(filepath)
+  } catch {
+    return filepath
+  }
+}
+
 function findExportByName (pkg, name) {
   for (const [key, value] of Object.entries(pkg)) {
     if (value?.name === name) {
@@ -95,8 +117,7 @@ function getProvidedContext () {
       _ddIsEarlyFlakeDetectionEnabled,
       _ddIsDiEnabled,
       _ddTestPropertiesByFilepath: testPropertiesByFilepath,
-      _ddEarlyFlakeDetectionNumRetries: numRepeats,
-      _ddEarlyFlakeDetectionSlowTestRetries: slowTestRetries,
+      _ddEarlyFlakeDetectionRetryPolicy: earlyFlakeDetectionRetryPolicy,
       _ddIsKnownTestsEnabled: isKnownTestsEnabled,
       _ddIsTestManagementTestsEnabled: isTestManagementTestsEnabled,
       _ddTestManagementAttemptToFixRetries: testManagementAttemptToFixRetries,
@@ -110,14 +131,18 @@ function getProvidedContext () {
       _ddTestCommand: testCommand,
       _ddRepositoryRoot: repositoryRoot,
       _ddCodeOwnersEntries: codeOwnersEntries,
+      _ddIsCodeCoverageEnabled: isCodeCoverageEnabled,
+      _ddItrCorrelationId: itrCorrelationId,
+      _ddUnskippableSuites: unskippableSuites,
+      _ddForcedToRunSuites: forcedToRunSuites,
     } = globalThis.__vitest_worker__.providedContext
+    const retryPolicy = earlyFlakeDetectionRetryPolicy ?? EMPTY_EFD_RETRY_POLICY
 
     return {
       isDiEnabled: _ddIsDiEnabled,
-      isEarlyFlakeDetectionEnabled: _ddIsEarlyFlakeDetectionEnabled,
+      isEarlyFlakeDetectionEnabled: _ddIsEarlyFlakeDetectionEnabled && hasEfdRetries(retryPolicy),
       testPropertiesByFilepath,
-      numRepeats,
-      slowTestRetries: slowTestRetries ?? {},
+      earlyFlakeDetectionRetryPolicy: retryPolicy,
       isKnownTestsEnabled,
       isTestManagementTestsEnabled,
       testManagementAttemptToFixRetries,
@@ -131,6 +156,10 @@ function getProvidedContext () {
       testCommand,
       repositoryRoot,
       codeOwnersEntries,
+      isCodeCoverageEnabled,
+      itrCorrelationId,
+      unskippableSuites,
+      forcedToRunSuites,
     }
   } catch {
     log.error('Vitest workers could not parse provided context, so some features will not work.')
@@ -138,8 +167,7 @@ function getProvidedContext () {
       isDiEnabled: false,
       isEarlyFlakeDetectionEnabled: false,
       testPropertiesByFilepath: {},
-      numRepeats: 0,
-      slowTestRetries: {},
+      earlyFlakeDetectionRetryPolicy: EMPTY_EFD_RETRY_POLICY,
       isKnownTestsEnabled: false,
       isTestManagementTestsEnabled: false,
       testManagementAttemptToFixRetries: 0,
@@ -153,6 +181,10 @@ function getProvidedContext () {
       testCommand: undefined,
       repositoryRoot: undefined,
       codeOwnersEntries: undefined,
+      isCodeCoverageEnabled: false,
+      itrCorrelationId: undefined,
+      unskippableSuites: {},
+      forcedToRunSuites: {},
     }
   }
 }
@@ -230,8 +262,11 @@ module.exports = {
   testManagementTestsCh,
   modifiedFilesCh,
   workerReportTraceCh,
+  workerReportCoverageCh,
   workerReportLogsCh,
+  workerReportTelemetryCh,
   codeCoverageReportCh,
+  realpath,
   findExportByName,
   getTestRunnerExport,
   getTypeTasks,

package/packages/datadog-instrumentations/src/vitest-worker.js

@@ -1,11 +1,15 @@
 'use strict'
 
+const path = require('node:path')
 const { performance } = require('node:perf_hooks')
+const { fileURLToPath } = require('node:url')
 
 const shimmer = require('../../datadog-shimmer')
+const log = require('../../dd-trace/src/log')
+const { getEfdRetryCountForDuration } = require('../../dd-trace/src/ci-visibility/efd-retry-policy')
 const {
   DYNAMIC_NAME_RE,
-  getEfdRetryCount,
+  getTestSuitePath,
   recordAttemptToFixExecution,
   logAttemptToFixTestExecution,
 } = require('../../dd-trace/src/plugins/util/test')
@@ -27,6 +31,7 @@ const {
   getTypeTasks,
   getTestName,
   getProvidedContext,
+  realpath,
   isFlakyTestRetriesEnabledForTask,
   getVitestTestProperties,
 } = require('./vitest-util')
@@ -52,10 +57,187 @@ const efdExecutionStartByTask = new WeakMap()
 const efdSkippedRetryResults = new WeakMap()
 const attemptToFixExecutions = new Map()
 const loggedAttemptToFixTests = new Set()
-const switchedStatuses = new WeakSet()
+const switchedStatuses = new WeakMap()
 let vitestGetFn = null
 let vitestSetFn = null
 let vitestGetHooks = null
+let preciseCoverageSession
+let isPreciseCoverageUnavailable = false
+let vitestCoverageSnapshot
+const wrappedCoverageWorkerStates = new WeakSet()
+const nonIsolatedCoverageFiles = new Set()
+
+/**
+ * Sends a command to a Node.js inspector session.
+ *
+ * @param {import('node:inspector').Session} session
+ * @param {string} method
+ * @param {object} [params]
+ * @returns {Promise<object>}
+ */
+function postInspectorCommand (session, method, params) {
+  return new Promise((resolve, reject) => {
+    session.post(method, params, (error, result) => {
+      if (error) {
+        reject(error)
+      } else {
+        resolve(result || {})
+      }
+    })
+  })
+}
+
+async function startPreciseCoverage () {
+  if (preciseCoverageSession) return true
+  if (isPreciseCoverageUnavailable) return false
+
+  let session
+  try {
+    const inspector = require('node:inspector')
+    session = new inspector.Session()
+    session.connect()
+    await postInspectorCommand(session, 'Profiler.enable')
+    await postInspectorCommand(session, 'Profiler.startPreciseCoverage', {
+      callCount: false,
+      detailed: false,
+    })
+    preciseCoverageSession = session
+    return true
+  } catch (error) {
+    isPreciseCoverageUnavailable = true
+    try {
+      session?.disconnect()
+    } catch {}
+    log.warn('Could not start Vitest TIA code coverage: %s', error?.message)
+    return false
+  }
+}
+
+function getCoverageFilename (url) {
+  if (!url) return
+
+  if (url.startsWith('file://')) {
+    try {
+      return fileURLToPath(url)
+    } catch {
+      return
+    }
+  }
+
+  if (path.isAbsolute(url)) return url
+}
+
+function isFileInRepository (filename, repositoryRoot) {
+  const relativeFilename = path.relative(repositoryRoot, filename)
+  return relativeFilename !== '..' &&
+    !relativeFilename.startsWith(`..${path.sep}`) &&
+    !path.isAbsolute(relativeFilename) &&
+    !relativeFilename.startsWith(`node_modules${path.sep}`) &&
+    !relativeFilename.includes(`${path.sep}node_modules${path.sep}`)
+}
+
+function isV8ScriptCovered (scriptCoverage) {
+  for (const functionCoverage of scriptCoverage.functions) {
+    for (const range of functionCoverage.ranges) {
+      if (range.count > 0) return true
+    }
+  }
+  return false
+}
+
+function getCoveredFilesFromV8Result (coverage, repositoryRoot) {
+  const coveredFiles = []
+  const scriptCoverageResults = coverage?.result
+  if (scriptCoverageResults) {
+    for (const scriptCoverage of scriptCoverageResults) {
+      if (!isV8ScriptCovered(scriptCoverage)) continue
+
+      const coverageFilename = getCoverageFilename(scriptCoverage.url)
+      if (!coverageFilename) continue
+
+      const filename = realpath(coverageFilename)
+      if (isFileInRepository(filename, repositoryRoot)) {
+        coveredFiles.push(filename)
+      }
+    }
+  }
+  return coveredFiles
+}
+
+function getVitestCoverageOptions () {
+  return globalThis.__vitest_worker__?.config?.coverage
+}
+
+/**
+ * Check whether the current Vitest worker reuses its module cache across test suites.
+ *
+ * @returns {boolean}
+ */
+function isNonIsolatedRun () {
+  const config = globalThis.__vitest_worker__?.config
+  if (config?.isolate === false) return true
+
+  return config?.poolOptions?.[config.pool]?.isolate === false
+}
+
+/**
+ * Conservatively include files covered by earlier suites when Vitest reuses its module cache.
+ *
+ * @param {string[] | undefined} coverageFiles
+ * @returns {string[] | undefined}
+ */
+function includePreviouslyCoveredFiles (coverageFiles) {
+  if (!coverageFiles || !isNonIsolatedRun()) return coverageFiles
+
+  // TODO: Track module cache hits per suite instead; cumulative coverage can under-skip with isolate:false.
+  for (const filename of coverageFiles) {
+    nonIsolatedCoverageFiles.add(filename)
+  }
+  return [...nonIsolatedCoverageFiles]
+}
+
+function usesVitestV8CoverageSnapshot (coverageOptions) {
+  return coverageOptions?.enabled === true &&
+    coverageOptions.provider === 'v8'
+}
+
+async function getPreciseCoverageFiles (repositoryRoot) {
+  if (!await startPreciseCoverage()) return
+
+  try {
+    const coverage = await postInspectorCommand(preciseCoverageSession, 'Profiler.takePreciseCoverage')
+    return getCoveredFilesFromV8Result(coverage, repositoryRoot)
+  } catch (error) {
+    isPreciseCoverageUnavailable = true
+    try {
+      preciseCoverageSession.disconnect()
+    } catch {}
+    preciseCoverageSession = undefined
+    log.warn('Could not collect Vitest TIA code coverage: %s', error?.message)
+  }
+}
+
+function getVitestCoverageFiles (repositoryRoot) {
+  return getCoveredFilesFromV8Result(vitestCoverageSnapshot, repositoryRoot)
+}
+
+function wrapVitestCoverageRpc () {
+  const workerState = globalThis.__vitest_worker__
+  if (!workerState?.rpc || wrappedCoverageWorkerStates.has(workerState)) return
+
+  wrappedCoverageWorkerStates.add(workerState)
+  workerState.rpc = new Proxy(workerState.rpc, {
+    get (target, property, receiver) {
+      const value = Reflect.get(target, property, receiver)
+      if (property !== 'onAfterSuiteRun' || typeof value !== 'function') return value
+
+      return function (metadata) {
+        vitestCoverageSnapshot = metadata?.coverage
+        return value.apply(this, arguments)
+      }
+    },
+  })
+}
 
 function waitForHitProbe () {
   const promises = {}
@@ -208,7 +390,7 @@ function wrapVitestTestRunner (VitestTestRunner) {
     const {
       isEarlyFlakeDetectionEnabled,
       isKnownTestsEnabled,
-      numRepeats,
+      earlyFlakeDetectionRetryPolicy,
       isTestManagementTestsEnabled,
       testManagementAttemptToFixRetries,
       isImpactedTestsEnabled,
@@ -247,7 +429,7 @@ function wrapVitestTestRunner (VitestTestRunner) {
       if (isEarlyFlakeDetectionEnabled) {
         efdRetryTasks.add(task)
         disableFrameworkRetries(task)
-        task.repeats = numRepeats
+        task.repeats = earlyFlakeDetectionRetryPolicy.schedulingRetryCount
       }
       modifiedTasks.add(task)
       taskToStatuses.set(task, [])
@@ -257,7 +439,7 @@ function wrapVitestTestRunner (VitestTestRunner) {
       if (isEarlyFlakeDetectionEnabled && !modifiedTasks.has(task)) {
         efdRetryTasks.add(task)
         disableFrameworkRetries(task)
-        task.repeats = numRepeats
+        task.repeats = earlyFlakeDetectionRetryPolicy.schedulingRetryCount
       }
       newTasks.add(task)
       taskToStatuses.set(task, [])
@@ -281,14 +463,14 @@ function wrapVitestTestRunner (VitestTestRunner) {
       if (isAttemptingToFix) {
         const statuses = attemptToFixTaskToStatuses.get(task)
         if (task.result.state === 'pass' && statuses?.includes('fail')) {
-          switchedStatuses.add(task)
+          switchedStatuses.set(task, task.result.state)
           task.result.state = 'fail'
         }
       }
 
       if (!isAttemptingToFix && isQuarantined) {
         if (task.result.state === 'fail') {
-          switchedStatuses.add(task)
+          switchedStatuses.set(task, task.result.state)
         }
         task.result.state = 'pass'
       }
@@ -299,7 +481,7 @@ function wrapVitestTestRunner (VitestTestRunner) {
       // If the test has passed at least once, we consider it passed
       if (statuses.includes('pass')) {
         if (task.result.state === 'fail') {
-          switchedStatuses.add(task)
+          switchedStatuses.set(task, task.result.state)
         }
         task.result.state = 'pass'
       }
@@ -321,7 +503,7 @@ function wrapVitestTestRunner (VitestTestRunner) {
       isKnownTestsEnabled,
       isEarlyFlakeDetectionEnabled,
       isDiEnabled,
-      slowTestRetries,
+      earlyFlakeDetectionRetryPolicy,
     } = providedContext
 
     if (isKnownTestsEnabled) {
@@ -337,7 +519,7 @@ function wrapVitestTestRunner (VitestTestRunner) {
       const duration = previousExecutionStart === undefined
         ? task.result?.duration ?? 0
         : performance.now() - previousExecutionStart
-      const retryCount = getEfdRetryCount(duration, slowTestRetries)
+      const retryCount = getEfdRetryCountForDuration(duration, earlyFlakeDetectionRetryPolicy)
       efdDeterminedRetries.set(task, retryCount)
       task.repeats = retryCount
       if (retryCount === 0) {
@@ -517,7 +699,7 @@ function wrapVitestTestRunner (VitestTestRunner) {
       const {
         isEarlyFlakeDetectionEnabled,
         testManagementAttemptToFixRetries,
-        slowTestRetries,
+        earlyFlakeDetectionRetryPolicy,
       } = getProvidedContext()
 
       const status = getVitestTestStatus(task, retryInfo.retry)
@@ -545,7 +727,7 @@ function wrapVitestTestRunner (VitestTestRunner) {
       ) {
         const executionStart = efdExecutionStartByTask.get(task)
         const duration = executionStart === undefined ? task.result?.duration ?? 0 : performance.now() - executionStart
-        const retryCount = getEfdRetryCount(duration, slowTestRetries)
+        const retryCount = getEfdRetryCountForDuration(duration, earlyFlakeDetectionRetryPolicy)
         efdDeterminedRetries.set(task, retryCount)
         task.repeats = retryCount
         if (retryCount === 0) {
@@ -649,6 +831,19 @@ addHook({
     // From >=3.0.1, the first arguments changes from a string to an object containing the filepath
     const testSuiteAbsolutePath = testPaths[0]?.filepath || testPaths[0]
     const providedContext = getProvidedContext()
+    const repositoryRoot = providedContext.repositoryRoot || process.cwd()
+    const testSuite = getTestSuitePath(testSuiteAbsolutePath, repositoryRoot)
+    const coverageOptions = getVitestCoverageOptions()
+    const shouldUseVitestCoverage = usesVitestV8CoverageSnapshot(coverageOptions)
+    const coverageLibrary = 'v8'
+    vitestCoverageSnapshot = undefined
+    if (providedContext.isCodeCoverageEnabled) {
+      if (shouldUseVitestCoverage) {
+        wrapVitestCoverageRpc()
+      } else {
+        await startPreciseCoverage()
+      }
+    }
 
     const testSuiteCtx = {
       testSuiteAbsolutePath,
@@ -656,8 +851,13 @@ addHook({
       testSessionId: providedContext.testSessionId,
       testModuleId: providedContext.testModuleId,
       testCommand: providedContext.testCommand,
-      repositoryRoot: providedContext.repositoryRoot,
+      repositoryRoot,
       codeOwnersEntries: providedContext.codeOwnersEntries,
+      isCodeCoverageEnabled: providedContext.isCodeCoverageEnabled,
+      coverageLibrary,
+      itrCorrelationId: providedContext.itrCorrelationId,
+      isUnskippable: providedContext.unskippableSuites?.[testSuite] === true,
+      isForcedToRun: providedContext.forcedToRunSuites?.[testSuite] === true,
     }
     testSuiteStartCh.runStores(testSuiteCtx, () => {})
     const startTestsResponse = await startTests.apply(this, arguments)
@@ -671,14 +871,15 @@ addHook({
       const { result } = task
       // We have to trick vitest into thinking that the test has passed
       // but we want to report it as failed if it did fail
-      const isSwitchedStatus = switchedStatuses.has(task)
+      const switchedStatus = switchedStatuses.get(task)
+      const isSwitchedStatus = switchedStatus !== undefined
 
       if (result) {
         const { state, duration, errors } = result
         const testError = getCurrentAttemptTestError(task, errors)
         if (attemptToFixTasks.has(task)) {
-          const status = getFinalAttemptToFixStatus(task, state, isSwitchedStatus, testCtx)
-          recordFinalAttemptToFixExecution(task, status, providedContext)
+          const attemptToFixStatus = getFinalAttemptToFixStatus(task, state, isSwitchedStatus, testCtx)
+          recordFinalAttemptToFixExecution(task, attemptToFixStatus, providedContext)
         }
 
         if (state === 'skip') { // programmatic skip
@@ -686,7 +887,9 @@ addHook({
             testName: getTestName(task),
             testSuiteAbsolutePath: task.file.filepath,
             isNew: newTasks.has(task),
+            isAttemptToFix: attemptToFixTasks.has(task),
             isDisabled: disabledTasks.has(task),
+            isQuarantined: quarantinedTasks.has(task),
           })
         } else if (state === 'pass' && !isSwitchedStatus) {
           if (testCtx) {
@@ -722,7 +925,8 @@ addHook({
             providedContext.isEarlyFlakeDetectionEnabled && (newTasks.has(task) || modifiedTasks.has(task))
           if (isEfdRetry) {
             const statuses = taskToStatuses.get(task)
-            const efdRetryCount = efdDeterminedRetries.get(task) ?? providedContext.numRepeats
+            const efdRetryCount = efdDeterminedRetries.get(task) ??
+              providedContext.earlyFlakeDetectionRetryPolicy.schedulingRetryCount
             // statuses only includes repetitions (not the initial run), so we check against retry count (not +1)
             if (efdRetryCount > 0 && statuses && statuses.length === efdRetryCount &&
               statuses.every(status => status === 'fail')) {
@@ -783,7 +987,9 @@ addHook({
           testName: getTestName(task),
           testSuiteAbsolutePath: task.file.filepath,
           isNew: newTasks.has(task),
+          isAttemptToFix: attemptToFixTasks.has(task),
           isDisabled: disabledTasks.has(task),
+          isQuarantined: quarantinedTasks.has(task),
         })
       }
     }
@@ -807,8 +1013,19 @@ addHook({
       testSuiteErrorCh.runStores(testSuiteCtx, () => {})
     }
 
+    let coverageFiles
+    if (providedContext.isCodeCoverageEnabled) {
+      const currentCoverageFiles = shouldUseVitestCoverage
+        ? getVitestCoverageFiles(repositoryRoot)
+        : await getPreciseCoverageFiles(repositoryRoot)
+      coverageFiles = includePreviouslyCoveredFiles(currentCoverageFiles)
+    }
+
     await getChannelPromise(testSuiteFinishCh, {
       status: testSuiteResult.state,
+      coverageFiles,
+      coverageLibrary,
+      testSuiteAbsolutePath,
       ...testSuiteCtx.currentStore,
     })