dd-trace 6.7.0 → 6.9.0

package/ci/test-optimization-validation/runner-contract.js

@@ -7,10 +7,10 @@ const { environmentNamesEqual, setEnvironmentValue } = require('./environment')
 
 const RUNNER_OPTIONS = {
   cucumber: {
-    flags: new Set(),
+    flags: new Set(['--backtrace', '--fail-fast', '--no-strict', '--strict']),
     values: new Set([
       '-p', '--config', '--import', '--language', '--loader', '--profile', '--require', '--require-module',
-      '--world-parameters',
+      '--parallel', '--world-parameters',
     ]),
   },
   cypress: {
@@ -18,7 +18,7 @@ const RUNNER_OPTIONS = {
     values: new Set(['--browser', '--config-file']),
   },
   jest: {
-    flags: new Set(['--detectLeaks']),
+    flags: new Set(['--color', '--colors', '--detectLeaks']),
     values: new Set(['-c', '--config', '--env', '--runner', '--testEnvironment']),
   },
   mocha: {
@@ -128,17 +128,6 @@ function getRunnerContract (framework, command, projectRoot, repositoryRoot, pla
     }
   }
   const omittedOptions = getOmittedRunnerOptions(framework, invocation)
-  if (framework === 'cucumber') {
-    const language = getOptionValue(invocation, '--language')
-    if (language && language !== 'en') {
-      return {
-        environment: {},
-        error: `--language ${language} is not supported by the validator-generated English scenarios`,
-        inputFiles: [],
-        runnerArgs: [],
-      }
-    }
-  }
 
   const environment = getRunnerEnvironment(invocation, platform)
   const environmentError = getRunnerEnvironmentError(environment, platform)
@@ -167,6 +156,23 @@ function getRunnerContract (framework, command, projectRoot, repositoryRoot, pla
   }
 }
 
+function getRunnerConfigurationContract (framework, runnerArgs, environment, projectRoot, repositoryRoot) {
+  const runnerArgsError = getRunnerArgsError(framework, runnerArgs)
+  if (runnerArgsError) {
+    return {
+      environment: {},
+      error: `runner arguments ${runnerArgsError}`,
+      inputFiles: [],
+      runnerArgs: [],
+    }
+  }
+  const inputs = getRunnerInputs(runnerArgs, environment, projectRoot, repositoryRoot)
+  if (inputs.error) {
+    return { environment: {}, error: inputs.error, inputFiles: [], runnerArgs: [] }
+  }
+  return { environment, inputFiles: inputs.files, runnerArgs }
+}
+
 /**
  * Rejects shell-expanded allowlisted environment assignments before command tokenization.
  *
@@ -239,7 +245,7 @@ function getRunnerSearchRoots (framework, command, projectRoot, repositoryRoot)
  */
 function addRunnerSearchRoot (roots, value, projectRoot, repositoryRoot, directory) {
   if (typeof value !== 'string' || !value) return
-  const literal = value.split(/[*?{[]/, 1)[0].replace(/[\\/]+$/, '')
+  const literal = value.split(/[*?{[]/, 1)[0].replace(/(?<![\\/])[\\/]+$/, '')
   if (!literal) return
   const candidate = path.resolve(projectRoot, literal)
   const searchRoot = directory || !path.extname(candidate) ? candidate : path.dirname(candidate)
@@ -305,17 +311,24 @@ function getUnknownRunnerOption (framework, invocation) {
 
 function getOmittedRunnerOptions (framework, invocation) {
   const omitted = OMITTED_RUNNER_OPTIONS[framework] || new Set()
+  const ignored = IGNORED_RUNNER_OPTIONS[framework] || { flags: new Set(), values: new Set() }
   return [...new Set(invocation.tokens
     .slice(invocation.runnerIndex + 1)
-    .filter(token => token.startsWith('-') && omitted.has(token.split('=', 1)[0])))]
+    .filter(token => {
+      if (!token.startsWith('-')) return false
+      const option = token.split('=', 1)[0]
+      return omitted.has(option) || ignored.flags.has(option) || ignored.values.has(option)
+    })
+    .map(token => token.split('=', 1)[0]))]
 }
 
-function getOptionValue (invocation, expected) {
-  const tokens = invocation.tokens.slice(invocation.runnerIndex + 1)
-  for (let index = 0; index < tokens.length; index++) {
-    if (tokens[index].split('=', 1)[0] !== expected) continue
-    return tokens[index].includes('=') ? tokens[index].slice(tokens[index].indexOf('=') + 1) : tokens[index + 1]
+function getArgumentOptionValues (args, expected) {
+  const values = []
+  for (let index = 0; index < args.length; index++) {
+    if (args[index].split('=', 1)[0] !== expected) continue
+    values.push(args[index].includes('=') ? args[index].slice(args[index].indexOf('=') + 1) : args[index + 1])
   }
+  return values
 }
 
 /**
@@ -361,7 +374,8 @@ function getRunnerArgsError (framework, args) {
     }
     const option = token.split('=', 1)[0]
     if (options.flags.has(option)) {
-      if (token !== option && !/^--detectLeaks=(?:true|false)$/.test(token)) {
+      if (token !== option &&
+        !/^--(?:color|colors|detectLeaks)=(?:true|false)$/.test(token)) {
         return `contain an invalid value for ${option}`
       }
       continue
@@ -376,6 +390,13 @@ function getRunnerArgsError (framework, args) {
       return `contain a missing or unsafe value for ${option}`
     }
   }
+
+  if (framework === 'cucumber') {
+    const language = getArgumentOptionValues(args, '--language').find(value => value !== 'en')
+    if (language) {
+      return `--language ${language} is not supported by the validator-generated English scenarios`
+    }
+  }
 }
 
 /**
@@ -411,10 +432,12 @@ function getRunnerInputError (args, environment, projectRoot, repositoryRoot, co
   if (inputs.error) return inputs.error
 
   const approved = new Set()
-  for (const filename of configFiles || []) {
-    try {
-      approved.add(fs.realpathSync(filename))
-    } catch {}
+  if (configFiles) {
+    for (const filename of configFiles) {
+      try {
+        approved.add(fs.realpathSync(filename))
+      } catch {}
+    }
   }
   const unbound = inputs.files.find(filename => !approved.has(filename))
   if (unbound) return `references an input that is not approval-bound: ${unbound}`
@@ -439,26 +462,34 @@ function getFrameworkInvocation (command, framework) {
   })
   if (runnerIndex === -1) return
 
-  const prefix = tokens.slice(0, runnerIndex)
-  const firstCommand = prefix.find(token => !/^[A-Za-z_][A-Za-z0-9_]*=/.test(token))
-  if (firstCommand && !RUNNER_LAUNCHERS.has(path.basename(firstCommand).toLowerCase())) {
-    return {
-      error: `runner launch wrapper ${path.basename(firstCommand)} is not allowlisted`,
-      runnerIndex,
-      tokens,
-    }
-  }
-  for (const token of prefix) {
-    if (RUNNER_LAUNCHERS.has(path.basename(token).toLowerCase())) continue
+  const prefixError = getRunnerPrefixError(tokens.slice(0, runnerIndex))
+  if (prefixError) return { error: prefixError, runnerIndex, tokens }
+  return { runnerIndex, tokens }
+}
+
+function getRunnerPrefixError (prefix) {
+  let coverageLauncher = false
+  for (let index = 0; index < prefix.length; index++) {
+    const token = prefix[index]
     if (/^[A-Za-z_][A-Za-z0-9_]*=[^;&|`]*$/.test(token)) continue
-    return {
-      error: 'runner launch wrapper contains options or positional arguments whose semantics the validator does not ' +
-        'preserve',
-      runnerIndex,
-      tokens,
+
+    const basename = path.basename(token).toLowerCase()
+    if (RUNNER_LAUNCHERS.has(basename)) {
+      coverageLauncher = basename === 'c8' || basename === 'nyc'
+      continue
     }
+    if (coverageLauncher && /^(?:-r|--reporter)=\S+$/.test(token)) continue
+    if (coverageLauncher && /^(?:-r|--reporter)$/.test(token) &&
+      prefix[index + 1] && !prefix[index + 1].startsWith('-')) {
+      index++
+      continue
+    }
+    if (!prefix.some(candidate => RUNNER_LAUNCHERS.has(path.basename(candidate).toLowerCase()))) {
+      return `runner launch wrapper ${path.basename(token)} is not allowlisted`
+    }
+    return 'runner launch wrapper contains options or positional arguments whose semantics the validator does not ' +
+      'preserve'
   }
-  return { runnerIndex, tokens }
 }
 
 /**
@@ -612,6 +643,10 @@ function resolveInputFile (input, projectRoot) {
   if (!input.module || path.isAbsolute(input.value) || input.value.startsWith('.')) {
     return path.resolve(projectRoot, input.value)
   }
+  const projectRelative = path.resolve(projectRoot, input.value)
+  try {
+    return require.resolve(projectRelative)
+  } catch {}
   try {
     return require.resolve(input.value, { paths: [projectRoot] })
   } catch {}
@@ -657,6 +692,7 @@ function isPathInside (root, filename) {
 module.exports = {
   getProjectNodeRunner,
   getRunnerArgsError,
+  getRunnerConfigurationContract,
   getRunnerContract,
   getRunnerEnvironmentError,
   getRunnerInputError,

package/ci/test-optimization-validation/safe-files.js

@@ -129,10 +129,10 @@ function openAndWrite (root, filename, data, label, creationFlag) {
  *
  * @param {string} directory directory path
  * @param {string} label customer-facing path label
- * @returns {{dev: number, ino: number}} directory identity
+ * @returns {{dev: bigint, ino: bigint}} directory identity
  */
 function getDirectoryIdentity (directory, label) {
-  const stat = fs.lstatSync(directory)
+  const stat = fs.lstatSync(directory, { bigint: true })
   if (!stat.isDirectory() || stat.isSymbolicLink()) {
     throw new Error(`Refusing ${label} because its parent is not a regular directory: ${directory}`)
   }
@@ -143,7 +143,7 @@ function getDirectoryIdentity (directory, label) {
  * Refuses replacement of a parent directory between safe creation and publication.
  *
  * @param {string} directory directory path
- * @param {{dev: number, ino: number}} expected expected identity
+ * @param {{dev: bigint, ino: bigint}} expected expected identity
  * @param {string} label customer-facing path label
  */
 function assertDirectoryIdentity (directory, expected, label) {
@@ -158,7 +158,7 @@ function assertDirectoryIdentity (directory, expected, label) {
  *
  * @param {string} filename temporary filename
  * @param {string} parent expected parent directory
- * @param {{dev: number, ino: number}} parentIdentity expected parent identity
+ * @param {{dev: bigint, ino: bigint}} parentIdentity expected parent identity
  */
 function removeTemporaryFile (filename, parent, parentIdentity) {
   try {

package/ci/test-optimization-validation/scenarios/basic-reporting.js

@@ -58,7 +58,7 @@ async function runBasicReporting ({ framework, out, options }) {
       selector,
       settingsLoadedFromCache: run.offline.inputs.settings?.status === 'loaded',
     }
-    evidence.foundationalReportingEstablished = evidence.foundationalReportingEstablished && selector.verified
+    evidence.foundationalReportingEstablished &&= selector.verified
 
     if (run.result.timedOut) {
       return inconclusive(
@@ -111,7 +111,7 @@ async function runBasicReporting ({ framework, out, options }) {
     }
 
     if (!run.offline.initialized || !evidence.settingsLoadedFromCache) {
-      return failWithDebugRerun({
+      return failBasicWithDebugRerun({
         command,
         diagnosis: run.offline.initialized
           ? 'The clean test passed, but Test Optimization did not load the validator-owned offline settings.'
@@ -131,7 +131,7 @@ async function runBasicReporting ({ framework, out, options }) {
     }
 
     if (run.result.exitCode !== 0) {
-      evidence.commandFailure = classifyFailure(framework, run.result)
+      evidence.commandFailure = classifyFailure(framework, run.result, run.events)
       const confirmation = await runCleanConfirmation({ command, framework, options, out })
       evidence.cleanConfirmation = confirmation.evidence
       if (!confirmation.evidence.matchesPreflight) {
@@ -144,7 +144,7 @@ async function runBasicReporting ({ framework, out, options }) {
           confirmation.artifacts
         )
       }
-      const result = await failWithDebugRerun({
+      const result = await failBasicWithDebugRerun({
         command,
         diagnosis: complete
           ? `The test passed twice without Datadog but exited ${run.result.exitCode} when initialized, after ` +
@@ -169,7 +169,7 @@ async function runBasicReporting ({ framework, out, options }) {
     }
 
     const missing = getMissingLevels(evidence)
-    return failWithDebugRerun({
+    return failBasicWithDebugRerun({
       command,
       diagnosis: 'The direct test passed cleanly and while initialized, but no complete Test Optimization event ' +
         `hierarchy was captured. Missing levels: ${missing.join(', ')}. This is a possible dd-trace adapter bug.`,
@@ -194,6 +194,50 @@ async function runBasicReporting ({ framework, out, options }) {
   }
 }
 
+async function failBasicWithDebugRerun (input) {
+  const result = await failWithDebugRerun(input)
+  const debug = result.evidence?.debugRerun
+  if (!isSuccessfulDebugRerun(debug, result.evidence?.selector)) return result
+
+  const initialDiagnosis = result.diagnosis
+  delete result.evidence.possibleLibraryBug
+  result.status = 'error'
+  result.diagnosis = 'The first initialized run failed, but the unchanged DD_TRACE_DEBUG=1 rerun exited cleanly ' +
+    'and emitted the complete event hierarchy. Basic Reporting is intermittent, so no library-bug conclusion ' +
+    'was reached.'
+  result.evidence.initialInstrumentedDiagnosis = initialDiagnosis
+  result.evidence.intermittentInstrumentedResult = true
+  result.evidence.recommendation = 'Repeat the approved initialized command without changing its test selection. ' +
+    'If the failure recurs, send both initialized runs and the clean confirmations to engineering.'
+  result.evidence.validationIncomplete = true
+  return result
+}
+
+function isSuccessfulDebugRerun (debug, selector) {
+  return isDebugSelectorVerified(debug, selector) &&
+    debug?.ran === true &&
+    debug.commandExitCode === 0 &&
+    debug.commandTimedOut === false &&
+    debug.offlineExporterInitialized === true &&
+    debug.settingsLoadedFromCache === true &&
+    debug.testSessionEvents > 0 &&
+    debug.testModuleEvents > 0 &&
+    debug.testSuiteEvents > 0 &&
+    debug.testEvents > 0
+}
+
+// A wrapper rerun is comparable only when events identify the approved representative and no other file.
+function isDebugSelectorVerified (debug, selector) {
+  if (selector?.verified !== true) return false
+  if (selector.mode === 'bounded_direct_runner') return true
+  if (selector.mode !== 'instrumented_event_identity' ||
+    typeof selector.expectedTestFile !== 'string' ||
+    debug?.testEventsWithoutSourceFile !== 0 ||
+    debug?.testSourceFileCount !== 1 ||
+    debug?.testSourceFiles?.length !== 1) return false
+  return sourceFilesMatch(debug.testSourceFiles[0], selector.expectedTestFile)
+}
+
 /**
  * Repeats the direct clean command after an initialized-only failure.
  *
@@ -234,10 +278,17 @@ async function runCleanConfirmation ({ command, framework, options, out }) {
  */
 function summarizePreflight (preflight = {}) {
   return {
+    attempts: (preflight.attempts || []).map(attempt => ({
+      exitCode: attempt.exitCode,
+      observedTestCount: attempt.observedTestCount,
+      testFile: attempt.testFile,
+      timedOut: attempt.timedOut === true,
+    })),
     durationMs: preflight.durationMs,
     exitCode: preflight.exitCode,
     observedTestCount: preflight.observedTestCount,
     ran: preflight.ran === true,
+    selectedTestFile: preflight.selectedTestFile,
     selectorVerification: preflight.selectorVerification,
     timedOut: preflight.timedOut === true,
   }
@@ -312,12 +363,15 @@ function normalizeSourceFile (filename) {
  *
  * @param {object} framework framework entry
  * @param {object} result command result
+ * @param {object[]} [events] normalized captured events
  * @returns {object|undefined} blocker
  */
-function classifyFailure (framework, result) {
+function classifyFailure (framework, result, events = []) {
   return getCommandBlocker(result, {
     browserRequired: framework.browserRequired,
     framework: framework.framework,
+    packageJson: framework.project?.packageJson,
+    testsRan: eventsOfType(events, 'test').length > 0,
   })
 }