dd-trace 6.4.0 → 6.5.0

package/ci/test-optimization-validation/safe-files.js

@@ -0,0 +1,203 @@
+'use strict'
+
+const crypto = require('node:crypto')
+const fs = require('node:fs')
+const path = require('node:path')
+
+/**
+ * Creates a directory while refusing symlink components and paths outside the allowed root.
+ *
+ * @param {string} root allowed root
+ * @param {string} directory directory to create or validate
+ * @param {string} label customer-facing path label
+ * @param {{allowRootSymlink?: boolean}} [options] validation options
+ */
+function ensureSafeDirectory (root, directory, label, options = {}) {
+  const lexicalRoot = path.resolve(root)
+  const resolvedDirectory = path.resolve(directory)
+  const rootStat = fs.lstatSync(lexicalRoot)
+  if (rootStat.isSymbolicLink() && !options.allowRootSymlink) {
+    throw new Error(`Refusing ${label} because its allowed root is a symbolic link: ${lexicalRoot}`)
+  }
+  if (!rootStat.isDirectory() && !rootStat.isSymbolicLink()) {
+    throw new Error(`Refusing ${label} because its allowed root is not a directory: ${lexicalRoot}`)
+  }
+  if (!isPathInside(lexicalRoot, resolvedDirectory)) {
+    throw new Error(`Refusing ${label} outside allowed root: ${resolvedDirectory}`)
+  }
+
+  let current = lexicalRoot
+  const relative = path.relative(lexicalRoot, resolvedDirectory)
+  for (const segment of relative ? relative.split(path.sep) : []) {
+    current = path.join(current, segment)
+    let stat
+    try {
+      stat = fs.lstatSync(current)
+    } catch (error) {
+      if (error.code !== 'ENOENT') throw error
+      fs.mkdirSync(current)
+      stat = fs.lstatSync(current)
+    }
+
+    if (stat.isSymbolicLink()) {
+      throw new Error(`Refusing ${label} through symbolic link: ${current}`)
+    }
+    if (!stat.isDirectory()) {
+      throw new Error(`Refusing ${label} through non-directory path: ${current}`)
+    }
+  }
+
+  const physicalRoot = fs.realpathSync(lexicalRoot)
+  const physicalDirectory = fs.realpathSync(resolvedDirectory)
+  if (!isPathInside(physicalRoot, physicalDirectory)) {
+    throw new Error(`Refusing ${label} outside physical allowed root: ${resolvedDirectory}`)
+  }
+}
+
+/**
+ * Writes a regular file without following a symbolic-link target.
+ *
+ * @param {string} root allowed root
+ * @param {string} filename output filename
+ * @param {string|Buffer} data output data
+ * @param {string} label customer-facing path label
+ */
+function writeFileSafely (root, filename, data, label) {
+  const resolvedFilename = path.resolve(filename)
+  const parent = path.dirname(resolvedFilename)
+  ensureSafeDirectory(root, parent, label)
+  const parentIdentity = getDirectoryIdentity(parent, label)
+  const temporaryFilename = path.join(
+    parent,
+    `.${path.basename(filename)}.${crypto.randomBytes(12).toString('hex')}.tmp`
+  )
+
+  try {
+    openAndWrite(root, temporaryFilename, data, label, fs.constants.O_EXCL)
+    assertDirectoryIdentity(parent, parentIdentity, label)
+    fs.renameSync(temporaryFilename, resolvedFilename)
+  } catch (error) {
+    removeTemporaryFile(temporaryFilename, parent, parentIdentity)
+    throw error
+  }
+}
+
+/**
+ * Creates a new regular file without following or replacing an existing path.
+ *
+ * @param {string} root allowed root
+ * @param {string} filename output filename
+ * @param {string|Buffer} data output data
+ * @param {string} label customer-facing path label
+ */
+function createFileSafely (root, filename, data, label) {
+  openAndWrite(root, filename, data, label, fs.constants.O_EXCL)
+}
+
+/**
+ * Opens and writes a file with no-follow semantics.
+ *
+ * @param {string} root allowed root
+ * @param {string} filename output filename
+ * @param {string|Buffer} data output data
+ * @param {string} label customer-facing path label
+ * @param {number} creationFlag file creation mode
+ */
+function openAndWrite (root, filename, data, label, creationFlag) {
+  const resolvedFilename = path.resolve(filename)
+  ensureSafeDirectory(root, path.dirname(resolvedFilename), label)
+  refuseSymbolicLink(resolvedFilename, label)
+
+  const flags = fs.constants.O_WRONLY |
+    fs.constants.O_CREAT |
+    creationFlag |
+    (fs.constants.O_NOFOLLOW || 0)
+  const file = fs.openSync(resolvedFilename, flags, 0o600)
+  try {
+    const stat = fs.fstatSync(file)
+    if (!stat.isFile() || stat.nlink !== 1) {
+      throw new Error(`Refusing ${label} because its output is not a private regular file: ${resolvedFilename}`)
+    }
+    fs.writeFileSync(file, data)
+  } finally {
+    fs.closeSync(file)
+  }
+}
+
+/**
+ * Captures one directory identity without accepting a symbolic link.
+ *
+ * @param {string} directory directory path
+ * @param {string} label customer-facing path label
+ * @returns {{dev: number, ino: number}} directory identity
+ */
+function getDirectoryIdentity (directory, label) {
+  const stat = fs.lstatSync(directory)
+  if (!stat.isDirectory() || stat.isSymbolicLink()) {
+    throw new Error(`Refusing ${label} because its parent is not a regular directory: ${directory}`)
+  }
+  return { dev: stat.dev, ino: stat.ino }
+}
+
+/**
+ * Refuses replacement of a parent directory between safe creation and publication.
+ *
+ * @param {string} directory directory path
+ * @param {{dev: number, ino: number}} expected expected identity
+ * @param {string} label customer-facing path label
+ */
+function assertDirectoryIdentity (directory, expected, label) {
+  const current = getDirectoryIdentity(directory, label)
+  if (current.dev !== expected.dev || current.ino !== expected.ino) {
+    throw new Error(`Refusing ${label} because its parent directory changed during the write: ${directory}`)
+  }
+}
+
+/**
+ * Removes only the validator-created temporary regular file or symbolic link.
+ *
+ * @param {string} filename temporary filename
+ * @param {string} parent expected parent directory
+ * @param {{dev: number, ino: number}} parentIdentity expected parent identity
+ */
+function removeTemporaryFile (filename, parent, parentIdentity) {
+  try {
+    assertDirectoryIdentity(parent, parentIdentity, 'temporary file cleanup')
+    const stat = fs.lstatSync(filename)
+    if (stat.isFile() || stat.isSymbolicLink()) fs.unlinkSync(filename)
+  } catch {}
+}
+
+/**
+ * Refuses a final path component that is a symbolic link.
+ *
+ * @param {string} filename candidate filename
+ * @param {string} label customer-facing path label
+ */
+function refuseSymbolicLink (filename, label) {
+  try {
+    if (fs.lstatSync(filename).isSymbolicLink()) {
+      throw new Error(`Refusing ${label} symbolic-link target: ${filename}`)
+    }
+  } catch (error) {
+    if (error.code !== 'ENOENT') throw error
+  }
+}
+
+/**
+ * Checks lexical path containment.
+ *
+ * @param {string} root allowed root
+ * @param {string} filename candidate path
+ * @returns {boolean} true when the candidate is inside the root
+ */
+function isPathInside (root, filename) {
+  const relative = path.relative(root, filename)
+  return relative === '' || (relative && !relative.startsWith('..') && !path.isAbsolute(relative))
+}
+
+module.exports = {
+  createFileSafely,
+  ensureSafeDirectory,
+  writeFileSafely,
+}

package/ci/test-optimization-validation/scenarios/auto-test-retries.js

@@ -0,0 +1,159 @@
+'use strict'
+
+const {
+  discoverScenarioTests,
+  discoveryEvidence,
+  error,
+  failWithDebugRerun,
+  pass,
+  prepareGeneratedScenario,
+  requireGeneratedScenario,
+  runInstrumentedCommand,
+  testEventSamples,
+  testsForDiscoveredScenario,
+} = require('./helpers')
+
+async function runAutoTestRetries ({ framework, out, options }) {
+  const scenarioName = 'atr'
+  const skipResult = requireGeneratedScenario(framework, 'atr-fail-once', scenarioName)
+  if (skipResult) return skipResult
+
+  let outDir
+  try {
+    const { scenario } = await prepareGeneratedScenario(framework, 'atr-fail-once')
+    const discovery = await discoverScenarioTests({ framework, out, scenarioName, scenario, options })
+    if (discovery.tests.length === 0) {
+      return failWithDebugRerun({
+        command: scenario.runCommand,
+        diagnosis: 'The fail-once generated test was not reported during baseline identity discovery.',
+        evidence: discoveryEvidence(discovery),
+        framework,
+        options,
+        out,
+        outDir: discovery.outDir,
+        scenarioName,
+      })
+    }
+
+    const fixtureConfig = {
+      settings: {
+        flaky_test_retries_enabled: true,
+      },
+    }
+
+    const run = await runInstrumentedCommand({
+      framework,
+      out,
+      scenarioName,
+      command: scenario.runCommand,
+      options,
+      fixtureConfig,
+    })
+    outDir = run.outDir
+
+    const tests = testsForDiscoveredScenario(run.events, scenario, discovery)
+    const autoTestRetryEvents = tests.filter(test => test.retryReason === 'auto_test_retry')
+    const externalRetryEvents = tests.filter(test => test.isRetry && test.retryReason !== 'auto_test_retry')
+    const evidence = {
+      ...discoveryEvidence(discovery),
+      commandExitCode: run.result.exitCode,
+      settingsLoadedFromCache: run.offline.inputs.settings?.status === 'loaded',
+      matchingTestEvents: tests.length,
+      autoTestRetryEvents: autoTestRetryEvents.length,
+      externalRetryEvents: externalRetryEvents.length,
+      failedAttempts: tests.filter(test => test.testStatus === 'fail' || test.error === 1).length,
+      passedAttempts: tests.filter(test => test.testStatus === 'pass').length,
+      samples: testEventSamples(tests),
+    }
+
+    if (!evidence.settingsLoadedFromCache) {
+      return failWithDebugRerun({
+        command: scenario.runCommand,
+        fixtureConfig,
+        diagnosis: 'Auto Test Retries settings were not loaded from the offline cache fixture.',
+        evidence,
+        framework,
+        options,
+        out,
+        outDir,
+        scenarioName,
+      })
+    }
+
+    if (run.result.exitCode !== 0) {
+      return failWithDebugRerun({
+        command: scenario.runCommand,
+        fixtureConfig,
+        diagnosis: getAutoTestRetriesFailureDiagnosis(framework, evidence),
+        evidence,
+        framework,
+        options,
+        out,
+        outDir,
+        scenarioName,
+      })
+    }
+
+    if (tests.length < 2 || autoTestRetryEvents.length === 0) {
+      return failWithDebugRerun({
+        command: scenario.runCommand,
+        fixtureConfig,
+        diagnosis: getAutoTestRetriesFailureDiagnosis(framework, evidence),
+        evidence,
+        framework,
+        options,
+        out,
+        outDir,
+        scenarioName,
+      })
+    }
+
+    return pass(
+      framework,
+      scenarioName,
+      'The fail-once generated test was retried and the command passed.',
+      evidence,
+      outDir
+    )
+  } catch (err) {
+    return error(framework, scenarioName, err, outDir)
+  }
+}
+
+function getAutoTestRetriesFailureDiagnosis (framework, evidence) {
+  const frameworkName = getFrameworkName(framework)
+  const retryTagSummary = getRetryTagSummary(evidence.autoTestRetryEvents)
+  if (evidence.autoTestRetryEvents > 0 || evidence.failedAttempts > 1) {
+    return 'Auto Test Retries executed for the generated test, but every attempt failed. Observed ' +
+      `${formatAttemptCount(evidence.failedAttempts, 'failed')}, ` +
+      `${formatAttemptCount(evidence.passedAttempts, 'passed retry')}, and ${retryTagSummary}. ` +
+      'Review the generated test failure because retry execution itself was observed.'
+  }
+  return 'Auto Test Retries was enabled, and the generated failing test was reported, but ' +
+    `${frameworkName} ` +
+    `did not execute a retry attempt. Observed ${formatAttemptCount(evidence.failedAttempts, 'failed')}, ` +
+    `${formatAttemptCount(evidence.passedAttempts, 'passed retry')}, and ${retryTagSummary}.`
+}
+
+function formatAttemptCount (count, label) {
+  return `${count} ${label} attempt${count === 1 ? '' : 's'}`
+}
+
+function getRetryTagSummary (count) {
+  if (count === 0) return 'no test.retry_reason=auto_test_retry tag'
+  if (count === 1) return '1 event tagged with test.retry_reason=auto_test_retry'
+  return `${count} events tagged with test.retry_reason=auto_test_retry`
+}
+
+function getFrameworkName (framework) {
+  return {
+    cucumber: 'Cucumber',
+    cypress: 'Cypress',
+    jest: 'Jest',
+    mocha: 'Mocha',
+    playwright: 'Playwright',
+    vitest: 'Vitest',
+  }[framework.framework] || 'the test runner'
+}
+
+module.exports = { getAutoTestRetriesFailureDiagnosis, runAutoTestRetries }