dd-trace 5.116.0 → 5.118.0

package/ci/test-optimization-validation/generated-test-contract.js

@@ -0,0 +1,283 @@
+'use strict'
+
+const path = require('node:path')
+
+const cucumberAdapter = require('./framework-adapters/cucumber')
+const cypressAdapter = require('./framework-adapters/cypress')
+const playwrightAdapter = require('./framework-adapters/playwright')
+
+const GENERATED_SCENARIOS = {
+  'basic-pass': {
+    expectedWithoutDatadog: { exitCode: 0, observedTestCount: 1 },
+    purpose: 'basic_reporting|efd_candidate',
+    suiteName: 'dd-test-optimization-validation',
+    testName: 'basic-pass',
+  },
+  'atr-fail-once': {
+    expectedWithoutDatadog: { exitCode: 1, observedTestCount: 1 },
+    purpose: 'auto_test_retries_candidate',
+    suiteName: 'dd-test-optimization-validation',
+    testName: 'atr-fail-once',
+  },
+  'test-management-target': {
+    expectedWithoutDatadog: { exitCode: 0, observedTestCount: 1 },
+    purpose: 'test_management_candidate',
+    suiteName: 'dd-test-optimization-validation',
+    testName: 'test-management-target',
+  },
+}
+
+/**
+ * Returns the retry-state path used by generated Node.js test scenarios.
+ *
+ * @param {string} framework test framework
+ * @param {string} filename generated ATR test file
+ * @returns {string} retry-state path
+ */
+function getGeneratedRetryStatePath (framework, filename) {
+  const namespace = `dd-test-optimization-validation-${framework}-`
+  const namespaced = path.basename(filename).includes(namespace) ||
+    path.basename(path.dirname(filename)).includes(namespace)
+  const stateFilename = namespaced
+    ? `.dd-test-optimization-validation-${framework}-atr-state`
+    : '.dd-test-optimization-validation-atr-state'
+  return path.join(path.dirname(filename), stateFilename)
+}
+
+/**
+ * Returns validator-owned source for one generated scenario.
+ *
+ * @param {object} input generated source input
+ * @param {string} input.framework test framework
+ * @param {string} input.moduleSystem generated module system
+ * @param {string} input.scenarioId generated scenario id
+ * @param {string} [input.stateFile] persistent ATR state file
+ * @returns {string} canonical generated source
+ */
+function getGeneratedTestContent ({ framework, moduleSystem, scenarioId, stateFile }) {
+  const scenario = GENERATED_SCENARIOS[scenarioId]
+  if (!scenario) throw new Error(`Unknown generated Test Optimization scenario: ${scenarioId}`)
+  if (framework === 'cucumber') {
+    return cucumberAdapter.getGeneratedTestContent({ testName: scenario.testName })
+  }
+  if (framework === 'cypress') {
+    return cypressAdapter.getGeneratedTestContent({
+      scenarioId,
+      testName: scenario.testName,
+    })
+  }
+  if (framework === 'playwright') {
+    return playwrightAdapter.getGeneratedTestContent({
+      moduleSystem,
+      scenarioId,
+      testName: scenario.testName,
+    })
+  }
+
+  const imports = []
+  if (framework === 'jest') {
+    const requireCall = 'require'
+    imports.push(moduleSystem === 'esm'
+      ? "import { describe, expect, test } from '@jest/globals'"
+      : `const { describe, expect, test } = ${requireCall}('@jest/globals')`)
+  }
+  if (framework === 'vitest' && moduleSystem === 'esm') {
+    imports.push("import { describe, expect, it } from 'vitest'")
+  }
+  if (framework === 'mocha') {
+    imports.push(moduleSystem === 'esm'
+      ? "import assert from 'node:assert/strict'"
+      : "const assert = require('node:assert/strict')")
+  }
+  if (scenarioId === 'atr-fail-once') {
+    if (moduleSystem === 'esm') {
+      imports.push("import { existsSync, writeFileSync } from 'node:fs'")
+    } else {
+      imports.push("const fs = require('node:fs')")
+    }
+  }
+
+  const assertion = framework === 'mocha' ? 'assert.equal(1, 1)' : 'expect(true).toBe(true)'
+  const testFunction = framework === 'jest' ? 'test' : 'it'
+  const body = scenarioId === 'atr-fail-once'
+    ? getAtrBody({ moduleSystem, assertion, stateFile })
+    : `    ${assertion}`
+
+  return [
+    ...imports,
+    imports.length > 0 ? '' : undefined,
+    "describe('dd-test-optimization-validation', () => {",
+    `  ${testFunction}('${scenario.testName}', () => {`,
+    body,
+    '  })',
+    '})',
+  ].filter(line => line !== undefined).join('\n')
+}
+
+/**
+ * Verifies that a runnable framework retained the validator-owned generated-test recipe.
+ *
+ * @param {object} framework manifest framework entry
+ * @returns {string|undefined} contract error
+ */
+function getGeneratedTestContractError (framework) {
+  const strategy = framework.generatedTestStrategy
+  if (!strategy || !['planned', 'verified'].includes(strategy.status)) return
+
+  if (!['cucumber', 'cypress', 'jest', 'mocha', 'playwright', 'vitest'].includes(framework.framework)) return
+  if (!['commonjs', 'esm'].includes(strategy.moduleSystem)) {
+    return 'must retain generatedTestStrategy.moduleSystem as "commonjs" or "esm".'
+  }
+
+  const files = strategy.files || []
+  const scenarios = strategy.scenarios || []
+  const projectRoot = framework.project?.root
+  if (typeof projectRoot !== 'string') return 'must declare a project root for generated tests.'
+  if ((strategy.cleanupPaths || []).some(filename => typeof filename !== 'string')) {
+    return 'must contain only string cleanup paths.'
+  }
+  const cleanupPaths = new Set((strategy.cleanupPaths || []).map(filename => path.normalize(filename)))
+  const expectedFileCount = ['cucumber', 'playwright'].includes(framework.framework) ? 4 : 3
+  if (files.length !== expectedFileCount || scenarios.length !== 3) {
+    return 'must contain exactly one validator-owned file for each of basic-pass, atr-fail-once, and ' +
+      `test-management-target${getAdditionalGeneratedFileDescription(framework.framework)}.`
+  }
+
+  const selectedFiles = new Set()
+  for (const [scenarioId, definition] of Object.entries(GENERATED_SCENARIOS)) {
+    const scenario = scenarios.find(entry => entry.id === scenarioId)
+    if (!scenario) return `is missing the validator-owned ${scenarioId} scenario.`
+    if (scenario.testIdentities?.length !== 1) {
+      return `scenario ${scenarioId} must declare exactly one generated test identity.`
+    }
+
+    const filename = scenario.testIdentities[0].file
+    if (typeof filename !== 'string') return `scenario ${scenarioId} must identify a generated test file.`
+    if (!isPathInside(projectRoot, filename)) {
+      return `scenario ${scenarioId} file must remain inside project root ${projectRoot}.`
+    }
+    const file = files.find(entry => {
+      return typeof entry.path === 'string' && path.normalize(entry.path) === path.normalize(filename)
+    })
+    if (!file) return `scenario ${scenarioId} must identify exactly one file in generatedTestStrategy.files.`
+    selectedFiles.add(path.normalize(filename))
+    if (scenario.testIdentities[0].name !== definition.testName) {
+      return `scenario ${scenarioId} must retain test name ${JSON.stringify(definition.testName)}.`
+    }
+
+    const stateFile = ['cucumber', 'cypress', 'playwright'].includes(framework.framework)
+      ? undefined
+      : getGeneratedRetryStatePath(framework.framework, filename)
+    const expectedSource = getGeneratedTestContent({
+      framework: framework.framework,
+      moduleSystem: strategy.moduleSystem,
+      scenarioId,
+      stateFile,
+    })
+    if (file.contentLines?.join('\n') !== expectedSource) {
+      return `scenario ${scenarioId} source differs from the validator-owned ${framework.framework} recipe. ` +
+        'Regenerate the manifest scaffold instead of rewriting temporary tests.'
+    }
+    if (!cleanupPaths.has(path.normalize(filename))) {
+      return `scenario ${scenarioId} file must be included in generatedTestStrategy.cleanupPaths.`
+    }
+  }
+
+  if (framework.framework === 'cucumber') {
+    const stepsFile = cucumberAdapter.getGeneratedStepsPath(strategy.testDirectory)
+    const steps = files.find(file => path.normalize(file.path) === path.normalize(stepsFile))
+    if (steps?.contentLines?.join('\n') !== cucumberAdapter.getGeneratedStepsContent()) {
+      return 'must retain the validator-owned Cucumber step definitions.'
+    }
+    if (!cleanupPaths.has(path.normalize(stepsFile))) {
+      return 'must include the isolated Cucumber step definitions in cleanupPaths.'
+    }
+  }
+
+  if (framework.framework === 'playwright') {
+    const configPath = playwrightAdapter.getGeneratedConfigPath(strategy.testDirectory)
+    const config = files.find(file => path.normalize(file.path) === path.normalize(configPath))
+    if (config?.contentLines?.join('\n') !== playwrightAdapter.getGeneratedConfigContent()) {
+      return 'must retain the validator-owned isolated Playwright config.'
+    }
+    if (!cleanupPaths.has(path.normalize(configPath))) {
+      return 'must include the isolated Playwright config in generatedTestStrategy.cleanupPaths.'
+    }
+  }
+
+  const usesPersistentRetryState = !['cucumber', 'cypress', 'playwright'].includes(framework.framework)
+  if (usesPersistentRetryState) {
+    const atrScenario = scenarios.find(entry => entry.id === 'atr-fail-once')
+    const stateFile = getGeneratedRetryStatePath(framework.framework, atrScenario.testIdentities[0].file)
+    if (!cleanupPaths.has(path.normalize(stateFile))) {
+      return `scenario atr-fail-once must clean up its persistent retry state file ${stateFile}.`
+    }
+  }
+  const expectedCleanupPathCount = usesPersistentRetryState ? 4 : expectedFileCount
+  if (selectedFiles.size !== 3 || cleanupPaths.size !== expectedCleanupPathCount) {
+    if (framework.framework === 'cucumber') {
+      return 'must use three distinct generated feature files and clean up those files plus the isolated Cucumber ' +
+        'step definitions.'
+    }
+    return usesPersistentRetryState
+      ? 'must use three distinct generated test files and clean up exactly those files plus the persistent ATR ' +
+        'state file.'
+      : 'must use three distinct generated test files and clean up exactly those files.'
+  }
+}
+
+/**
+ * Describes adapter-specific files in generated-contract errors.
+ *
+ * @param {string} framework framework name
+ * @returns {string} additional file description
+ */
+function getAdditionalGeneratedFileDescription (framework) {
+  if (framework === 'cucumber') return ' plus isolated Cucumber step definitions'
+  if (framework === 'playwright') return ' plus one isolated Playwright config'
+  return ''
+}
+
+/**
+ * Checks whether a generated path is inside or equal to the declared project root.
+ *
+ * @param {string} root project root
+ * @param {string} filename generated path
+ * @returns {boolean} whether the path is contained
+ */
+function isPathInside (root, filename) {
+  const relative = path.relative(path.resolve(root), path.resolve(filename))
+  return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative))
+}
+
+/**
+ * Returns the persistent retry body used across independent test-runner processes.
+ *
+ * @param {object} input body input
+ * @param {string} input.moduleSystem generated module system
+ * @param {string} input.assertion passing assertion
+ * @param {string} input.stateFile persistent ATR state file
+ * @returns {string} retry test body
+ */
+function getAtrBody ({ moduleSystem, assertion, stateFile }) {
+  if (typeof stateFile !== 'string' || !path.isAbsolute(stateFile)) {
+    throw new Error('The validator-owned ATR recipe requires an absolute persistent state file path.')
+  }
+  const exists = moduleSystem === 'esm' ? 'existsSync' : 'fs.existsSync'
+  const write = moduleSystem === 'esm' ? 'writeFileSync' : 'fs.writeFileSync'
+  return [
+    `    const stateFile = ${JSON.stringify(stateFile)}`,
+    `    if (!${exists}(stateFile)) {`,
+    `      ${write}(stateFile, 'failed-once')`,
+    "      throw new Error('dd-test-optimization-validation atr first failure')",
+    '    }',
+    `    ${assertion}`,
+  ].join('\n')
+}
+
+module.exports = {
+  GENERATED_SCENARIOS,
+  getGeneratedRetryStatePath,
+  getGeneratedTestContent,
+  getGeneratedTestContractError,
+}

package/ci/test-optimization-validation/generated-verifier.js

@@ -3,15 +3,13 @@
 const fs = require('node:fs')
 const path = require('node:path')
 
+const { getCommandBlocker } = require('./command-blocker')
 const { runCommand, serializeDisplayCommand } = require('./command-runner')
 const {
   cleanupGeneratedRuntimeFiles,
   writeGeneratedFiles,
 } = require('./generated-files')
-const {
-  getDatadogCleanCommand,
-  getLocalValidationCommand,
-} = require('./local-command')
+const { getGeneratedCommand } = require('./runner-command')
 const { frameworkOutDir } = require('./scenarios/helpers')
 const { getObservedTestCount } = require('./test-output')
 
@@ -42,11 +40,11 @@ async function verifyGeneratedTestStrategy ({ framework, out, options }) {
 
   try {
     cleanupGeneratedRuntimeFiles(framework)
-    writeGeneratedFiles(framework)
 
     for (const scenario of getScenariosToVerify(strategy.scenarios, options.scenarios)) {
       cleanupGeneratedRuntimeFiles(framework)
-      const command = getDatadogCleanCommand(getLocalValidationCommand(framework, scenario.runCommand))
+      writeGeneratedFiles(framework, scenario)
+      const command = getGeneratedCommand(framework, scenario)
       const outDir = frameworkOutDir(out, framework, `generated-verification-${scenario.id}`)
       // Generated commands run serially because fail-once state and cleanup are scenario-local.
       // eslint-disable-next-line no-await-in-loop
@@ -62,7 +60,7 @@ async function verifyGeneratedTestStrategy ({ framework, out, options }) {
       const observedTestCount = getObservedTestCount(framework.framework, result.stdout, result.stderr)
       const expected = scenario.expectedWithoutDatadog
       const failOnceStateCreated = scenario.id === 'atr-fail-once'
-        ? hasGeneratedRuntimeFile(strategy)
+        ? getGeneratedRuntimeFileStatus(strategy)
         : undefined
       const scenarioEvidence = {
         id: scenario.id,
@@ -77,10 +75,19 @@ async function verifyGeneratedTestStrategy ({ framework, out, options }) {
       evidence.scenarios.push(scenarioEvidence)
       artifacts.push(...Object.values(result.artifacts))
 
+      const observedWrongTestCount = observedTestCount !== null &&
+        observedTestCount !== expected.observedTestCount
       if (result.timedOut || result.exitCode !== expected.exitCode ||
-        observedTestCount !== expected.observedTestCount || failOnceStateCreated === false) {
+        observedWrongTestCount || failOnceStateCreated === false) {
         cleanupGeneratedRuntimeFiles(framework)
-        return getVerificationFailure(framework, evidence, artifacts, scenarioEvidence, result.timedOut)
+        return getVerificationFailure(
+          framework,
+          evidence,
+          artifacts,
+          scenarioEvidence,
+          result,
+          observedTestCount
+        )
       }
     }
 
@@ -138,12 +145,36 @@ function getScenariosToVerify (scenarios, selectedFeatures) {
  * @param {object} evidence collected verification evidence
  * @param {string[]} artifacts command artifacts
  * @param {object} scenario failed scenario evidence
- * @param {boolean} timedOut whether the scenario timed out
+ * @param {object} result command result
+ * @param {number|null} observedTestCount observed generated test count
  * @returns {{ok: false, failure: object}} generated verification failure
  */
-function getVerificationFailure (framework, evidence, artifacts, scenario, timedOut) {
+function getVerificationFailure (framework, evidence, artifacts, scenario, result, observedTestCount) {
+  const commandFailure = getCommandBlocker(result, {
+    browserRequired: framework.browserRequired,
+    framework: framework.framework,
+    testsRan: Number.isInteger(observedTestCount) && observedTestCount > 0,
+  })
+  if (commandFailure) {
+    const domain = commandFailure.blockedByExecutionEnvironment
+      ? 'execution_environment'
+      : commandFailure.localRuntimeBlocked
+        ? 'local_runtime'
+        : 'project_setup'
+    return {
+      ok: false,
+      failure: {
+        frameworkId: framework.id,
+        scenario: 'generated-test-verification',
+        status: 'blocked',
+        diagnosis: `${commandFailure.summary} Advanced-feature validation could not start reliably.`,
+        evidence: { ...evidence, commandFailure, domain, validationIncomplete: true },
+        artifacts,
+      },
+    }
+  }
   let reason
-  if (timedOut) {
+  if (result.timedOut) {
     reason = 'timed out'
   } else if (scenario.id === 'atr-fail-once' && scenario.failOnceStateCreated === false) {
     reason = 'failed without creating its declared fail-once state file, so it failed for an unrelated reason'
@@ -165,22 +196,24 @@ function getVerificationFailure (framework, evidence, artifacts, scenario, timed
 }
 
 /**
- * Checks whether the generated fail-once scenario created a declared runtime state file.
+ * Checks a declared generated runtime state file when the adapter uses one.
  *
  * @param {object} strategy generated test strategy
- * @returns {boolean} whether a regular declared runtime file exists
+ * @returns {boolean|undefined} whether a declared runtime file exists, or undefined when none is required
  */
-function hasGeneratedRuntimeFile (strategy) {
+function getGeneratedRuntimeFileStatus (strategy) {
   const generatedFiles = new Set((strategy.files || []).map(file => path.resolve(file.path)))
+  let expectsRuntimeFile = false
   for (const cleanupPath of strategy.cleanupPaths || []) {
     const filename = path.resolve(cleanupPath)
     if (generatedFiles.has(filename)) continue
+    expectsRuntimeFile = true
     try {
       const stat = fs.lstatSync(filename)
       if (!stat.isSymbolicLink() && stat.isFile()) return true
     } catch {}
   }
-  return false
+  return expectsRuntimeFile ? false : undefined
 }
 
 /**

package/ci/test-optimization-validation/literal-environment.js

@@ -0,0 +1,57 @@
+'use strict'
+
+const { environmentNamesEqual } = require('./environment')
+
+const ASSIGNMENT_PATTERN =
+  /([A-Za-z_][A-Za-z0-9_]*)=(?:"([^"\r\n]*)"|'([^'\r\n]*)'|([^\s]*))(?:[ \t]+|$)/y
+
+/**
+ * Parses only complete literal environment assignments at the start of a command.
+ *
+ * @param {string} command command text
+ * @returns {{assignments: Array<{name: string, source: string, value: string}>, length: number}} parsed prefix
+ */
+function parseLiteralEnvironmentPrefix (command) {
+  const source = String(command || '')
+  const assignments = []
+  let offset = 0
+
+  while (offset < source.length) {
+    ASSIGNMENT_PATTERN.lastIndex = offset
+    const match = ASSIGNMENT_PATTERN.exec(source)
+    if (!match) break
+    assignments.push({
+      name: match[1],
+      source: match[0].trimEnd(),
+      value: match[2] ?? match[3] ?? match[4],
+    })
+    offset = ASSIGNMENT_PATTERN.lastIndex
+  }
+
+  return { assignments, length: offset }
+}
+
+/**
+ * Removes complete leading assignments that explicitly clear one environment variable.
+ *
+ * @param {string} command command text
+ * @param {string} name environment variable
+ * @param {string} [platform] target platform
+ * @returns {string} command without empty assignments
+ */
+function removeEmptyLiteralEnvironmentAssignments (command, name, platform = process.platform) {
+  const source = String(command || '')
+  const prefix = parseLiteralEnvironmentPrefix(source)
+  const retained = prefix.assignments.filter(assignment => {
+    return !environmentNamesEqual(assignment.name, name, platform) || assignment.value !== ''
+  })
+  if (retained.length === prefix.assignments.length) return source
+  return [...retained.map(assignment => assignment.source), source.slice(prefix.length)]
+    .filter(Boolean)
+    .join(' ')
+}
+
+module.exports = {
+  parseLiteralEnvironmentPrefix,
+  removeEmptyLiteralEnvironmentAssignments,
+}

package/ci/test-optimization-validation/manifest-loader.js

@@ -76,18 +76,14 @@ function getManifestPaths (manifest) {
     paths.push(
       [`${prefix}.project.root`, framework.project?.root],
       [`${prefix}.project.packageJson`, framework.project?.packageJson],
+      [`${prefix}.validation.runner`, framework.validation?.runner],
+      [`${prefix}.validation.testFile`, framework.validation?.testFile],
       [`${prefix}.ciWiring.configFile`, framework.ciWiring?.configFile],
       [`${prefix}.ciWiring.workingDirectory`, framework.ciWiring?.workingDirectory]
     )
     for (const [index, configFile] of (framework.project?.configFiles || []).entries()) {
       paths.push([`${prefix}.project.configFiles[${index}]`, configFile])
     }
-    for (const [name, command] of getCommands(framework)) {
-      paths.push([`${prefix}.${name}.cwd`, command.cwd])
-      for (const [outputIndex, outputPath] of (command.outputPaths || []).entries()) {
-        paths.push([`${prefix}.${name}.outputPaths[${outputIndex}]`, outputPath])
-      }
-    }
 
     const strategy = framework.generatedTestStrategy
     paths.push([`${prefix}.generatedTestStrategy.testDirectory`, strategy?.testDirectory])
@@ -109,22 +105,6 @@ function getManifestPaths (manifest) {
   return paths
 }
 
-function getCommands (framework) {
-  const commands = []
-  for (const name of ['existingTestCommand', 'ciWiringCommand']) {
-    if (framework[name]) commands.push([name, framework[name]])
-  }
-  for (const [index, command] of (framework.setup?.commands || []).entries()) {
-    commands.push([`setup.commands[${index}]`, command])
-  }
-  for (const [index, scenario] of (framework.generatedTestStrategy?.scenarios || []).entries()) {
-    if (scenario?.runCommand) {
-      commands.push([`generatedTestStrategy.scenarios[${index}].runCommand`, scenario.runCommand])
-    }
-  }
-  return commands
-}
-
 function isPathInside (root, filename) {
   const relative = path.relative(root, filename)
   return relative === '' || (relative && !relative.startsWith('..') && !path.isAbsolute(relative))