dd-trace 6.6.0 → 6.8.0

package/ci/test-optimization-validation/scenarios/early-flake-detection.js

@@ -7,6 +7,7 @@ const {
   failWithDebugRerun,
   pass,
   prepareGeneratedScenario,
+  reportMissingGeneratedTest,
   requireGeneratedScenario,
   runInstrumentedCommand,
   skip,
@@ -28,14 +29,14 @@ async function runEarlyFlakeDetection ({ framework, out, options }) {
 
     const discovery = await discoverScenarioTests({ framework, out, scenarioName, scenario, options })
     if (discovery.tests.length === 0) {
-      return failWithDebugRerun({
+      return reportMissingGeneratedTest({
         command: scenario.runCommand,
         diagnosis: 'The generated new-test candidate was not reported during baseline identity discovery.',
-        evidence: discoveryEvidence(discovery),
+        discovery,
         framework,
         options,
         out,
-        outDir: discovery.outDir,
+        scenario,
         scenarioName,
       })
     }

package/ci/test-optimization-validation/scenarios/helpers.js

@@ -4,7 +4,7 @@ const fs = require('node:fs')
 const path = require('path')
 
 const { getArtifactId } = require('../artifact-id')
-const { buildCiWiringEnv, buildDatadogEnv, runCommand } = require('../command-runner')
+const { buildDatadogEnv, buildOfflineCaptureEnv, runCommand } = require('../command-runner')
 const {
   cleanupGeneratedRuntimeFiles,
   findGeneratedScenario,
@@ -14,10 +14,10 @@ const {
   eventsOfType,
   findTestsByIdentity,
 } = require('../payload-normalizer')
-const { getLocalValidationCommand } = require('../local-command')
 const { cleanupOfflineFixture, createOfflineFixture } = require('../offline-fixtures')
 const { readOfflineOutput } = require('../offline-output')
 const { sanitizeForReport, sanitizeString } = require('../redaction')
+const { getGeneratedCommand } = require('../runner-command')
 const { ensureSafeDirectory, writeFileSafely } = require('../safe-files')
 
 const ANSI_PATTERN = new RegExp(`${String.fromCharCode(27)}${String.raw`\[[0-?]*[ -/]*[@-~]`}`, 'g')
@@ -34,7 +34,7 @@ async function runInstrumentedCommand ({
   options,
   extraEnv,
   fixtureConfig,
-  ciWiring = false,
+  injectInitialization = true,
   allowMissingInitialization = false,
 }) {
   const outDir = frameworkOutDir(out, framework, scenarioName)
@@ -52,9 +52,9 @@ async function runInstrumentedCommand ({
       scenarioName,
       ...fixtureConfig,
     })
-    const validationEnv = ciWiring
-      ? buildCiWiringEnv({ fixture, outputRoot: rawOutputRoot })
-      : buildDatadogEnv({ fixture, outputRoot: rawOutputRoot, scenario: scenarioName, framework })
+    const validationEnv = injectInitialization
+      ? buildDatadogEnv({ fixture, outputRoot: rawOutputRoot, scenario: scenarioName, framework })
+      : buildOfflineCaptureEnv({ fixture, outputRoot: rawOutputRoot })
     result = await runCommand(command, {
       env: {
         ...validationEnv,
@@ -82,7 +82,7 @@ async function runInstrumentedCommand ({
       `${JSON.stringify(sanitizeForReport(result), null, 2)}\n`,
       'scenario result artifact'
     )
-    if (!offline.initialized && !ciWiring && !allowMissingInitialization) {
+    if (!offline.initialized && injectInitialization && !allowMissingInitialization) {
       const stderr = sanitizeString(result.stderr).trim().slice(-2000)
       throw new Error(
         'Offline Test Optimization exporter did not initialize or write completion evidence. ' +
@@ -180,11 +180,11 @@ async function prepareGeneratedScenario (framework, scenarioId) {
   const scenario = findGeneratedScenario(framework, scenarioId)
   if (!scenario) return { scenario: null, written: [] }
   cleanupGeneratedRuntimeFiles(framework)
-  const written = await writeGeneratedFiles(framework)
+  const written = await writeGeneratedFiles(framework, scenario)
   return {
     scenario: {
       ...scenario,
-      runCommand: getLocalValidationCommand(framework, scenario.runCommand),
+      runCommand: getGeneratedCommand(framework, scenario),
     },
     written,
   }
@@ -314,6 +314,63 @@ async function discoverScenarioTests ({ framework, out, scenarioName, scenario,
   }
 }
 
+/**
+ * Reports a missing generated test without blaming dd-trace when clean execution was unproven.
+ *
+ * @param {object} input missing-test evidence
+ * @param {object} input.command generated scenario command
+ * @param {string} input.diagnosis diagnosis used when clean execution was proven
+ * @param {object} input.discovery instrumented baseline result
+ * @param {object} input.framework framework manifest entry
+ * @param {object} input.options execution options
+ * @param {string} input.out validation output root
+ * @param {object} input.scenario generated scenario
+ * @param {string} input.scenarioName advanced scenario name
+ * @returns {object|Promise<object>} incomplete or confirmed failure result
+ */
+function reportMissingGeneratedTest ({
+  command,
+  diagnosis,
+  discovery,
+  framework,
+  options,
+  out,
+  scenario,
+  scenarioName,
+}) {
+  const verification = framework.generatedTestStrategy?.verification?.observedScenarios
+    ?.find(observed => observed.id === scenario.id)
+  const evidence = {
+    ...discoveryEvidence(discovery),
+    generatedVerificationObservedTestCount: verification?.observedTestCount,
+  }
+  if (verification?.observedTestCount === null) {
+    return inconclusive(
+      framework,
+      scenarioName,
+      'The clean temporary validation command exited as expected without a parseable test count, and the ' +
+        'instrumented baseline emitted no matching test event. The validator cannot prove that the generated test ' +
+        'executed, so no advanced-feature conclusion was reached.',
+      {
+        ...evidence,
+        reasonCode: 'generated-test-execution-unproven',
+      },
+      discovery.outDir
+    )
+  }
+
+  return failWithDebugRerun({
+    command,
+    diagnosis,
+    evidence,
+    framework,
+    options,
+    out,
+    outDir: discovery.outDir,
+    scenarioName,
+  })
+}
+
 function testsForDiscoveredScenario (events, scenario, discovery) {
   if (discovery?.testIdentities?.length > 0) {
     return findTestsByIdentity(events, discovery.testIdentities)
@@ -528,6 +585,7 @@ module.exports = {
   inconclusive,
   pass,
   prepareGeneratedScenario,
+  reportMissingGeneratedTest,
   requireGeneratedScenario,
   runDebugInstrumentedCommand,
   runInstrumentedCommand,

package/ci/test-optimization-validation/scenarios/test-management.js

@@ -9,6 +9,7 @@ const {
   failWithDebugRerun,
   pass,
   prepareGeneratedScenario,
+  reportMissingGeneratedTest,
   requireGeneratedScenario,
   runInstrumentedCommand,
   testEventSamples,
@@ -25,14 +26,14 @@ async function runTestManagement ({ framework, out, options }) {
     const { scenario } = await prepareGeneratedScenario(framework, 'test-management-target')
     const discovery = await discoverScenarioTests({ framework, out, scenarioName, scenario, options })
     if (discovery.tests.length === 0) {
-      return failWithDebugRerun({
+      return reportMissingGeneratedTest({
         command: scenario.runCommand,
         diagnosis: 'The test-management target was not reported during baseline identity discovery.',
-        evidence: discoveryEvidence(discovery),
+        discovery,
         framework,
         options,
         out,
-        outDir: discovery.outDir,
+        scenario,
         scenarioName,
       })
     }
@@ -148,7 +149,7 @@ function buildQuarantinedResponse (framework, scenario, discoveredIdentities = [
   for (const identity of identities) {
     for (const suite of getSuiteCandidates(identity, scenario)) {
       for (const name of getNameCandidates(identity)) {
-        suites[suite] = suites[suite] || { tests: {} }
+        suites[suite] ||= { tests: {} }
         suites[suite].tests[name] = {
           properties: {
             quarantined: true,
@@ -209,7 +210,7 @@ function summarizeManagedTests (testManagementTests) {
     }
     summary.set(displaySuite, testNames)
   }
-  return [...summary.entries()].slice(0, 5).map(([suite, tests]) => ({
+  return [...summary].slice(0, 5).map(([suite, tests]) => ({
     suite,
     tests: [...tests].slice(0, 5),
   }))

package/ci/test-optimization-validation/source-text.js

@@ -0,0 +1,87 @@
+'use strict'
+
+/**
+ * Replaces JavaScript comments with spaces while preserving line structure and string literals.
+ *
+ * @param {string} source JavaScript or TypeScript source
+ * @returns {string} source with comments masked
+ */
+function maskJavaScriptComments (source) {
+  return maskJavaScriptSource(source, false)
+}
+
+/**
+ * Replaces JavaScript comments and string literals with spaces while preserving line structure.
+ *
+ * @param {string} source JavaScript or TypeScript source
+ * @returns {string} source with non-code text masked
+ */
+function maskJavaScriptNonCode (source) {
+  return maskJavaScriptSource(source, true)
+}
+
+/**
+ * Masks comments and optionally string literals in JavaScript-like source.
+ *
+ * @param {string} source JavaScript or TypeScript source
+ * @param {boolean} maskStrings whether string and template literal contents should be masked
+ * @returns {string} masked source
+ */
+function maskJavaScriptSource (source, maskStrings) {
+  const characters = [...source]
+  let blockComment = false
+  let lineComment = false
+  let quote = ''
+
+  for (let index = 0; index < characters.length; index++) {
+    const character = characters[index]
+    const next = characters[index + 1]
+    if (lineComment) {
+      if (character === '\n') {
+        lineComment = false
+      } else {
+        characters[index] = ' '
+      }
+      continue
+    }
+    if (blockComment) {
+      if (character === '*' && next === '/') {
+        characters[index] = ' '
+        characters[++index] = ' '
+        blockComment = false
+      } else if (character !== '\r' && character !== '\n') {
+        characters[index] = ' '
+      }
+      continue
+    }
+    if (quote) {
+      if (maskStrings && character !== '\r' && character !== '\n') characters[index] = ' '
+      if (character.charCodeAt(0) === 92) {
+        if (maskStrings && index + 1 < characters.length &&
+          characters[index + 1] !== '\r' && characters[index + 1] !== '\n') {
+          characters[index + 1] = ' '
+        }
+        index++
+      } else if (character === quote) {
+        quote = ''
+      }
+      continue
+    }
+    if (character === '"' || character === '\'' || character === '`') {
+      quote = character
+      if (maskStrings) characters[index] = ' '
+    } else if (character === '/' && next === '/') {
+      characters[index] = ' '
+      characters[++index] = ' '
+      lineComment = true
+    } else if (character === '/' && next === '*') {
+      characters[index] = ' '
+      characters[++index] = ' '
+      blockComment = true
+    }
+  }
+
+  return characters.join('')
+}
+
+module.exports = { maskJavaScriptComments, maskJavaScriptNonCode }

package/ci/test-optimization-validation/test-output.js

@@ -1,5 +1,9 @@
 'use strict'
 
+const cypressAdapter = require('./framework-adapters/cypress')
+const cucumberAdapter = require('./framework-adapters/cucumber')
+const playwrightAdapter = require('./framework-adapters/playwright')
+
 const ANSI_PATTERN = new RegExp(`${String.fromCharCode(27)}${String.raw`\[[0-?]*[ -/]*[@-~]`}`, 'g')
 
 /**
@@ -11,16 +15,14 @@ const ANSI_PATTERN = new RegExp(`${String.fromCharCode(27)}${String.raw`\[[0-?]*
  * @returns {number|null} observed count when a supported summary was found
  */
 function getObservedTestCount (framework, stdout = '', stderr = '') {
-  const output = `${stdout}\n${stderr}`.replaceAll(ANSI_PATTERN, '')
+  const output = stripAnsi(`${stdout}\n${stderr}`)
+  if (framework === 'cucumber') return cucumberAdapter.getObservedTestCount(output)
+  if (framework === 'cypress') return cypressAdapter.getObservedTestCount(output)
+  if (framework === 'playwright') return playwrightAdapter.getObservedTestCount(output)
   if (framework === 'jest') return getJestObservedTestCount(output)
   if (framework === 'vitest') return getVitestObservedTestCount(output)
-  if (framework === 'playwright') return getPlaywrightObservedTestCount(output)
 
-  const totalPatterns = framework === 'node:test'
-    ? [/^# tests\s+(\d+)\s*$/gim]
-    : framework === 'cucumber'
-      ? [/\b(\d+)\s+scenarios?\b/gi]
-      : []
+  const totalPatterns = framework === 'node:test' ? [/^# tests\s+(\d+)\s*$/gim] : []
 
   for (const pattern of totalPatterns) {
     const count = getLastMatchCount(output, pattern)
@@ -38,20 +40,8 @@ function getObservedTestCount (framework, stdout = '', stderr = '') {
   return getLastMatchCount(output, /\b(\d+)\s+tests?\s+(?:passed|failed)\b/gi)
 }
 
-/**
- * Counts Playwright tests that completed instead of treating skipped tests as executions.
- *
- * @param {string} output test output without ANSI codes
- * @returns {number|null} executed test count
- */
-function getPlaywrightObservedTestCount (output) {
-  const observed = sumLastMatchCounts(output, [
-    /^\s*(\d+)\s+passed\b/gim,
-    /^\s*(\d+)\s+failed\b/gim,
-    /^\s*(\d+)\s+flaky\b/gim,
-  ])
-  if (observed !== null) return observed
-  return /^\s*\d+\s+skipped\b/im.test(output) ? 0 : null
+function stripAnsi (value) {
+  return String(value || '').replaceAll(ANSI_PATTERN, '')
 }
 
 /**
@@ -126,4 +116,4 @@ function sumLastMatchCounts (output, patterns) {
   return found ? count : null
 }
 
-module.exports = { getObservedTestCount }
+module.exports = { getObservedTestCount, stripAnsi }