dd-trace 5.117.0 → 5.118.0

package/ci/test-optimization-validation/framework-adapters/cucumber.js

@@ -0,0 +1,119 @@
+'use strict'
+
+const path = require('node:path')
+
+const CUCUMBER_PACKAGE = '@cucumber/cucumber'
+const CONFIG_PATTERN = /^cucumber\.(?:[cm]?js|json|ya?ml)$/
+const GENERATED_STEPS_FILENAME = 'dd-test-optimization-validation.steps.cjs'
+
+/**
+ * Reports whether a file follows the Cucumber feature convention.
+ *
+ * @param {string} filename candidate filename
+ * @returns {boolean} whether the candidate can be selected by Cucumber
+ */
+function isTestFile (filename) {
+  return filename.endsWith('.feature')
+}
+
+/**
+ * Counts statically declared Cucumber scenarios in a feature.
+ *
+ * @param {string} source feature source
+ * @returns {number} declared scenario count
+ */
+function getScenarioCount (source) {
+  return [...source.matchAll(/^\s*Scenario(?: Outline)?:\s*\S/gm)].length
+}
+
+/**
+ * Returns validator-owned Cucumber feature source for one generated scenario.
+ *
+ * @param {object} input generated source input
+ * @param {string} input.testName generated scenario name
+ * @returns {string} canonical generated feature source
+ */
+function getGeneratedTestContent ({ testName }) {
+  return [
+    'Feature: Datadog Test Optimization validation',
+    '',
+    `  Scenario: ${testName}`,
+    `    Given the Datadog validation scenario ${JSON.stringify(testName)}`,
+  ].join('\n')
+}
+
+/**
+ * Returns the validator-owned Cucumber step definitions shared by generated scenarios.
+ *
+ * @returns {string} canonical generated step-definition source
+ */
+function getGeneratedStepsContent () {
+  return [
+    "'use strict'",
+    '',
+    `const { Given } = require(${JSON.stringify(CUCUMBER_PACKAGE)})`,
+    '',
+    'let atrAttempt = 0',
+    '',
+    "Given('the Datadog validation scenario {string}', function (scenario) {",
+    "  if (scenario === 'atr-fail-once' && atrAttempt++ === 0) {",
+    "    throw new Error('dd-test-optimization-validation atr first failure')",
+    '  }',
+    '})',
+  ].join('\n')
+}
+
+/**
+ * Returns the generated Cucumber step-definition path for a feature directory.
+ *
+ * @param {string} testDirectory generated feature directory
+ * @returns {string} generated step-definition path
+ */
+function getGeneratedStepsPath (testDirectory) {
+  return path.join(testDirectory, GENERATED_STEPS_FILENAME)
+}
+
+/**
+ * Returns Cucumber arguments that select one existing feature.
+ *
+ * @param {string} filename selected Cucumber feature
+ * @returns {string[]} focused Cucumber arguments
+ */
+function getFocusedTestArgs (filename) {
+  return [filename, '--format', 'progress']
+}
+
+/**
+ * Returns Cucumber arguments for one isolated generated scenario.
+ *
+ * @param {string} filename generated Cucumber feature
+ * @param {string} stepsFile generated Cucumber step definitions
+ * @returns {string[]} generated scenario arguments
+ */
+function getGeneratedTestArgs (filename, stepsFile) {
+  return ['--require', stepsFile, '--format', 'progress', filename]
+}
+
+/**
+ * Extracts the executed-scenario count from the final Cucumber summary.
+ *
+ * @param {string} output Cucumber output without ANSI escapes
+ * @returns {number|null} final scenario count when present
+ */
+function getObservedTestCount (output) {
+  let count = null
+  for (const match of output.matchAll(/\b(\d+)\s+scenarios?\b/gi)) count = Number(match[1])
+  return count
+}
+
+module.exports = {
+  CONFIG_PATTERN,
+  getFocusedTestArgs,
+  getGeneratedStepsContent,
+  getGeneratedStepsPath,
+  getGeneratedTestArgs,
+  getGeneratedTestContent,
+  getObservedTestCount,
+  getScenarioCount,
+  isTestFile,
+}

package/ci/test-optimization-validation/framework-adapters/cypress.js

@@ -0,0 +1,107 @@
+'use strict'
+
+const path = require('node:path')
+
+const CONFIG_PATTERN = /^cypress(?:\.config)?\.(?:[cm]?[jt]s|json)$/
+const TEST_FILE_PATTERN = /(?:\.cy\.[cm]?[jt]sx?|\.(?:spec|test)\.[cm]?[jt]sx?)$/
+
+/**
+ * Reports whether a file follows a Cypress spec convention.
+ *
+ * @param {string} filename candidate filename
+ * @param {string} directory candidate parent directory
+ * @param {string} projectRoot detected project root
+ * @returns {boolean} whether the candidate can be selected by Cypress
+ */
+function isTestFile (filename, directory, projectRoot) {
+  if (/\.cy\.[cm]?[jt]sx?$/.test(filename)) return true
+  const relativeDirectory = new Set(path.relative(projectRoot, directory).split(path.sep))
+  return relativeDirectory.has('cypress') &&
+    (relativeDirectory.has('e2e') || relativeDirectory.has('integration')) &&
+    TEST_FILE_PATTERN.test(filename)
+}
+
+/**
+ * Returns the complete suffix that a generated Cypress spec must preserve.
+ *
+ * @param {string} filename representative Cypress spec
+ * @returns {string} generated spec suffix
+ */
+function getTestExtension (filename) {
+  return TEST_FILE_PATTERN.exec(path.basename(filename))?.[0] || '.cy.js'
+}
+
+/**
+ * Returns validator-owned Cypress source for one generated scenario.
+ *
+ * @param {object} input generated source input
+ * @param {string} input.scenarioId generated scenario id
+ * @param {string} input.testName generated test name
+ * @returns {string} canonical generated Cypress source
+ */
+function getGeneratedTestContent ({ scenarioId, testName }) {
+  const lines = []
+  if (scenarioId === 'atr-fail-once') {
+    lines.push('let attempt = 0', '')
+  }
+  lines.push(
+    "describe('dd-test-optimization-validation', () => {",
+    `  it(${JSON.stringify(testName)}, () => {`,
+    scenarioId === 'atr-fail-once'
+      ? '    expect(attempt++).to.equal(1)'
+      : '    expect(true).to.equal(true)',
+    '  })',
+    '})'
+  )
+  return lines.join('\n')
+}
+
+/**
+ * Returns Cypress arguments that select one existing project spec.
+ *
+ * @param {string} filename selected Cypress spec
+ * @returns {string[]} focused Cypress arguments
+ */
+function getFocusedTestArgs (filename) {
+  return ['--spec', filename]
+}
+
+/**
+ * Returns Cypress arguments for one isolated generated scenario.
+ *
+ * @param {string} filename generated Cypress spec
+ * @param {string[]} configurationArgs approved Cypress configuration arguments
+ * @returns {string[]} generated scenario arguments
+ */
+function getGeneratedTestArgs (filename, configurationArgs) {
+  return [
+    'run',
+    ...configurationArgs,
+    '--spec',
+    filename,
+    '--config',
+    'video=false,screenshotOnRunFailure=false,retries=0',
+  ]
+}
+
+/**
+ * Extracts the executed-test count from the final Cypress run summary.
+ *
+ * @param {string} output Cypress output without ANSI escapes
+ * @returns {number|null} final test count when present
+ */
+function getObservedTestCount (output) {
+  let count = null
+  for (const match of output.matchAll(/\bTests\s*:\s*(\d+)\b/gi)) count = Number(match[1])
+  return count
+}
+
+module.exports = {
+  CONFIG_PATTERN,
+  getFocusedTestArgs,
+  getGeneratedTestArgs,
+  getGeneratedTestContent,
+  getObservedTestCount,
+  getTestExtension,
+  isTestFile,
+}

package/ci/test-optimization-validation/framework-adapters/playwright.js

@@ -0,0 +1,181 @@
+'use strict'
+
+const path = require('node:path')
+
+const CONFIG_PATTERN = /^playwright\.config\.[cm]?[jt]s$/
+const TEST_FILE_PATTERN = /(?:\.(?:spec|test)\.[cm]?[jt]sx?)$/
+const GENERATED_CONFIG_FILENAME = 'dd-test-optimization-validation.playwright.config.cjs'
+const VALIDATION_OUTPUT_DIRECTORY = '.dd-test-optimization-validation-playwright-output'
+const PLAYWRIGHT_PACKAGE = '@playwright/test'
+
+/**
+ * Reports whether a file follows a Playwright Test convention.
+ *
+ * @param {string} filename candidate filename
+ * @param {string} directory candidate parent directory
+ * @param {string} projectRoot detected project root
+ * @returns {boolean} whether the candidate can be selected by Playwright Test
+ */
+function isTestFile (filename, directory, projectRoot) {
+  if (!TEST_FILE_PATTERN.test(filename)) return false
+  const relative = path.relative(projectRoot, path.join(directory, filename))
+  return relative !== '' && !relative.startsWith('..') && !path.isAbsolute(relative)
+}
+
+/**
+ * Returns the complete suffix that a generated Playwright spec must preserve.
+ *
+ * @param {string} filename representative Playwright spec
+ * @returns {string} generated spec suffix
+ */
+function getTestExtension (filename) {
+  return TEST_FILE_PATTERN.exec(path.basename(filename))?.[0] || '.spec.js'
+}
+
+/**
+ * Returns validator-owned Playwright source for one generated scenario.
+ *
+ * @param {object} input generated source input
+ * @param {string} input.moduleSystem generated module system
+ * @param {string} input.scenarioId generated scenario id
+ * @param {string} input.testName generated test name
+ * @returns {string} canonical generated Playwright source
+ */
+function getGeneratedTestContent ({ moduleSystem, scenarioId, testName }) {
+  const assertion = scenarioId === 'atr-fail-once'
+    ? '  expect(test.info().retry).toBe(1)'
+    : '  expect(true).toBe(true)'
+  const imports = moduleSystem === 'commonjs'
+    ? `const { expect, test } = require(${JSON.stringify(PLAYWRIGHT_PACKAGE)})`
+    : `import { expect, test } from ${JSON.stringify(PLAYWRIGHT_PACKAGE)}`
+  return [
+    imports,
+    '',
+    `test(${JSON.stringify(testName)}, async () => {`,
+    assertion,
+    '})',
+  ].join('\n')
+}
+
+/**
+ * Returns the isolated Playwright config shared by generated validation tests.
+ *
+ * @returns {string} canonical generated Playwright config
+ */
+function getGeneratedConfigContent () {
+  return [
+    `const { defineConfig } = require(${JSON.stringify(PLAYWRIGHT_PACKAGE)})`,
+    '',
+    'module.exports = defineConfig({',
+    '  fullyParallel: false,',
+    '  forbidOnly: true,',
+    "  reporter: 'line',",
+    '  retries: 0,',
+    '  testDir: __dirname,',
+    '  workers: 1,',
+    '})',
+  ].join('\n')
+}
+
+/**
+ * Returns the generated Playwright config path for a test directory.
+ *
+ * @param {string} testDirectory generated test directory
+ * @returns {string} generated config path
+ */
+function getGeneratedConfigPath (testDirectory) {
+  return path.join(testDirectory, GENERATED_CONFIG_FILENAME)
+}
+
+/**
+ * Returns Playwright arguments that select one existing project spec.
+ *
+ * @param {string} filename selected Playwright spec
+ * @returns {string[]} focused Playwright arguments
+ */
+function getFocusedTestArgs (filename) {
+  return [
+    filename,
+    '--reporter=line',
+    '--workers=1',
+    `--output=${getOutputPath(filename)}`,
+  ]
+}
+
+/**
+ * Returns Playwright arguments for one isolated generated scenario.
+ *
+ * @param {string} filename generated Playwright spec
+ * @param {string} configFile generated Playwright config
+ * @returns {string[]} generated scenario arguments
+ */
+function getGeneratedTestArgs (filename, configFile) {
+  return [
+    'test',
+    '--config',
+    configFile,
+    filename,
+    '--reporter=line',
+    '--workers=1',
+    `--output=${getOutputPath(filename)}`,
+  ]
+}
+
+/**
+ * Returns the validator-owned Playwright output directory for a selected spec.
+ *
+ * @param {string} filename selected or generated Playwright spec
+ * @returns {string} absolute output directory
+ */
+function getOutputPath (filename) {
+  return path.join(path.dirname(filename), VALIDATION_OUTPUT_DIRECTORY)
+}
+
+/**
+ * Extracts the executed-test count from the final Playwright summary.
+ *
+ * @param {string} output Playwright output without ANSI escapes
+ * @returns {number|null} executed test count when present
+ */
+function getObservedTestCount (output) {
+  const observed = sumLastMatchCounts(output, [
+    /^\s*(\d+)\s+passed\b/gim,
+    /^\s*(\d+)\s+failed\b/gim,
+    /^\s*(\d+)\s+flaky\b/gim,
+  ])
+  if (observed !== null) return observed
+  return /^\s*\d+\s+skipped\b/im.test(output) ? 0 : null
+}
+
+/**
+ * Sums the final count for each Playwright summary pattern.
+ *
+ * @param {string} output Playwright output
+ * @param {RegExp[]} patterns summary patterns
+ * @returns {number|null} summed count when any pattern matched
+ */
+function sumLastMatchCounts (output, patterns) {
+  let found = false
+  let total = 0
+  for (const pattern of patterns) {
+    let count
+    for (const match of output.matchAll(pattern)) count = Number(match[1])
+    if (count === undefined) continue
+    found = true
+    total += count
+  }
+  return found ? total : null
+}
+
+module.exports = {
+  CONFIG_PATTERN,
+  getFocusedTestArgs,
+  getGeneratedConfigContent,
+  getGeneratedConfigPath,
+  getGeneratedTestArgs,
+  getGeneratedTestContent,
+  getObservedTestCount,
+  getOutputPath,
+  getTestExtension,
+  isTestFile,
+}

package/ci/test-optimization-validation/generated-files.js

@@ -15,7 +15,7 @@ const initializedCleanupStrategies = new WeakSet()
 const authorizedRuntimeCleanupFiles = new Map()
 const writtenGeneratedFiles = new Map()
 
-function writeGeneratedFiles (framework) {
+function writeGeneratedFiles (framework, scenario) {
   const strategy = framework.generatedTestStrategy
   if (!strategy || !['planned', 'verified'].includes(strategy.status)) {
     return []
@@ -26,8 +26,9 @@ function writeGeneratedFiles (framework) {
   initializeRuntimeCleanupFiles(framework, strategy)
 
   const written = []
+  const files = getScenarioFiles(strategy, scenario)
   try {
-    for (const file of strategy.files || []) {
+    for (const file of files) {
       const filename = validateGeneratedFilePath(framework, file.path)
       validateContentLines(file.contentLines, filename)
       const content = `${file.contentLines.join('\n')}\n`
@@ -62,14 +63,47 @@ function writeGeneratedFiles (framework) {
   return written
 }
 
+/**
+ * Returns one scenario file plus adapter support files, or every file when no scenario is selected.
+ *
+ * @param {object} strategy generated strategy
+ * @param {object|undefined} scenario selected generated scenario
+ * @returns {object[]} generated files to write
+ */
+function getScenarioFiles (strategy, scenario) {
+  if (!scenario) return strategy.files || []
+  const scenarioPaths = new Set((strategy.scenarios || []).map(candidate => {
+    return path.resolve(candidate.testIdentities[0].file)
+  }))
+  const selectedPath = path.resolve(scenario.testIdentities[0].file)
+  return (strategy.files || []).filter(file => {
+    const filename = path.resolve(file.path)
+    return filename === selectedPath || !scenarioPaths.has(filename)
+  })
+}
+
 function cleanupGeneratedFiles (manifest, { keep = false } = {}) {
-  if (keep) return
+  if (keep) return { status: 'retained_by_request' }
 
+  const outcome = {
+    directoriesRemoved: 0,
+    directoriesRetained: 0,
+    filesRemoved: 0,
+    filesRetained: 0,
+  }
   for (const framework of manifest.frameworks || []) {
     const strategy = framework.generatedTestStrategy
-    cleanupPaths(getSafeCleanupPaths(framework, strategy, { includeGeneratedFiles: true }))
-    cleanupCreatedDirectories(framework.project.root)
+    addCleanupOutcome(
+      outcome,
+      cleanupPaths(getSafeCleanupPaths(framework, strategy, { includeGeneratedFiles: true })),
+      'files'
+    )
+    addCleanupOutcome(outcome, cleanupCreatedDirectories(framework.project.root), 'directories')
   }
+  outcome.status = outcome.filesRetained > 0 || outcome.directoriesRetained > 0
+    ? 'incomplete'
+    : 'completed'
+  return outcome
 }
 
 /**
@@ -96,21 +130,30 @@ function getMissingDirectories (root, directory) {
  * @param {string} root project root
  */
 function cleanupCreatedDirectories (root) {
+  const outcome = { removed: 0, retained: 0 }
   const resolvedRoot = path.resolve(root)
   const directories = [...createdGeneratedDirectories.entries()]
-    .filter(([directory, authorization]) => {
-      return isPathInside(resolvedRoot, directory) && isCleanupAuthorizationValid(directory, authorization)
-    })
+    .filter(([directory]) => isPathInside(resolvedRoot, directory))
     .sort(([left], [right]) => right.length - left.length)
 
-  for (const [directory] of directories) {
+  for (const [directory, authorization] of directories) {
+    if (!isCleanupAuthorizationValid(directory, authorization)) {
+      if (pathExists(directory)) outcome.retained++
+      continue
+    }
     try {
       fs.rmdirSync(directory)
       createdGeneratedDirectories.delete(directory)
+      outcome.removed++
     } catch (error) {
-      if (error.code === 'ENOENT') createdGeneratedDirectories.delete(directory)
+      if (error.code === 'ENOENT') {
+        createdGeneratedDirectories.delete(directory)
+      } else {
+        outcome.retained++
+      }
     }
   }
+  return outcome
 }
 
 function cleanupGeneratedRuntimeFiles (framework) {
@@ -166,18 +209,37 @@ function getSafeCleanupPaths (framework, strategy, { includeGeneratedFiles }) {
 }
 
 function cleanupPaths (cleanupPaths) {
+  const outcome = { removed: 0, retained: 0 }
   for (const cleanupPath of cleanupPaths) {
     try {
       const authorization = writtenGeneratedFiles.get(cleanupPath) ||
         authorizedRuntimeCleanupFiles.get(cleanupPath)
-      if (!authorization || !isCleanupAuthorizationValid(cleanupPath, authorization)) continue
-      if (isDirectory(cleanupPath)) continue
+      if (!authorization || !isCleanupAuthorizationValid(cleanupPath, authorization) || isDirectory(cleanupPath)) {
+        if (pathExists(cleanupPath)) outcome.retained++
+        continue
+      }
       fs.rmSync(cleanupPath, { force: true })
       writtenGeneratedFiles.delete(cleanupPath)
+      outcome.removed++
     } catch {
-      // Cleanup should be best-effort. The report will contain the command artifacts.
+      if (pathExists(cleanupPath)) outcome.retained++
     }
   }
+  return outcome
+}
+
+function addCleanupOutcome (target, outcome, kind) {
+  target[`${kind}Removed`] += outcome.removed
+  target[`${kind}Retained`] += outcome.retained
+}
+
+function pathExists (filename) {
+  try {
+    fs.lstatSync(filename)
+    return true
+  } catch {
+    return false
+  }
 }
 
 function forgetWrittenGeneratedFiles (filenames) {