dd-trace 6.4.0 → 6.5.0

package/ci/test-optimization-validation/scenarios/basic-reporting.js

@@ -0,0 +1,657 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+
+const { runCommand } = require('../command-runner')
+const { getDatadogCleanCommand, getLocalValidationCommand } = require('../local-command')
+
+const {
+  basicEventEvidence,
+  error,
+  failWithDebugRerun,
+  findInterestingLines,
+  frameworkOutDir,
+  hasAllBasicEventTypes,
+  inconclusive,
+  pass,
+  runInstrumentedCommand,
+  tailInterestingLines,
+} = require('./helpers')
+
+async function runBasicReporting ({ framework, out, options }) {
+  const scenarioName = 'basic-reporting'
+  try {
+    const command = getBasicReportingCommand(framework)
+    const { result, events, offline, outDir } = await runInstrumentedCommand({
+      framework,
+      out,
+      scenarioName,
+      command,
+      options,
+      allowMissingInitialization: true,
+    })
+
+    const evidence = {
+      commandExitCode: result.exitCode,
+      commandTimedOut: result.timedOut,
+      commandDescription: command?.description,
+      commandOutputSummary: summarizeTestOutput(result.stdout, result.stderr),
+      manifestNotes: Array.isArray(framework.notes) ? framework.notes : [],
+      preflight: summarizePreflight(framework.preflight),
+      offlineExporterInitialized: offline.initialized,
+      settingsLoadedFromCache: offline.inputs.settings?.status === 'loaded',
+      offlineExporterSummary: offline.summary,
+      ...basicEventEvidence(events),
+    }
+
+    if (evidence.offlineExporterInitialized && !evidence.settingsLoadedFromCache) {
+      return error(
+        framework,
+        scenarioName,
+        new Error('Basic Reporting settings were not loaded from the authoritative offline cache fixture.'),
+        outDir
+      )
+    }
+
+    if (!hasAllBasicEventTypes(events)) {
+      if (result.exitCode !== 0) {
+        evidence.commandFailure = summarizeCommandFailure(result, evidence)
+      }
+      const eventLevelFailure = getMissingEventDiagnosis({ framework, result, evidence })
+      evidence.eventLevelFailure = eventLevelFailure
+
+      return failBasicReportingWithDebugRerun({
+        command,
+        diagnosis: eventLevelFailure.summary,
+        evidence,
+        framework,
+        options,
+        out,
+        outDir,
+        scenarioName,
+        skipDebug: evidence.commandFailure?.buildErrors?.length > 0 ||
+          !shouldRunDebugRerun(eventLevelFailure, result),
+      })
+    }
+
+    if (result.exitCode === 0) {
+      return pass(
+        framework,
+        scenarioName,
+        'Basic reporting emitted session, module, suite, and test events.',
+        evidence,
+        outDir
+      )
+    }
+
+    if (matchesPreflightExitCode(framework.preflight, result.exitCode)) {
+      evidence.commandFailure = summarizeCommandFailure(result, evidence)
+      evidence.commandExitMatchesPreflight = true
+      return pass(
+        framework,
+        scenarioName,
+        'Basic reporting emitted session, module, suite, and test events. ' +
+          `The command exited ${result.exitCode}, matching the dd-trace-less preflight run.`,
+        evidence,
+        outDir
+      )
+    }
+
+    evidence.commandFailure = summarizeCommandFailure(result, evidence)
+    evidence.commandExitMatchesPreflight = false
+    const cleanConfirmation = await runCleanConfirmation({ command, framework, options, out, scenarioName })
+    evidence.cleanConfirmation = cleanConfirmation.evidence
+
+    if (!cleanConfirmation.evidence.exitMatchesPreflight) {
+      return inconclusive(
+        framework,
+        scenarioName,
+        getUnstableBaselineDiagnosis(framework.preflight, result, cleanConfirmation.result),
+        evidence,
+        outDir,
+        cleanConfirmation.artifacts
+      )
+    }
+
+    const failure = await failBasicReportingWithDebugRerun({
+      command,
+      diagnosis: getPossibleCompatibilityDiagnosis(framework.preflight, result),
+      evidence,
+      framework,
+      options,
+      out,
+      outDir,
+      scenarioName,
+    })
+    failure.artifacts.push(...cleanConfirmation.artifacts)
+    return failure
+  } catch (err) {
+    return error(framework, scenarioName, err)
+  }
+}
+
+/**
+ * Repeats the selected command without Datadog after an instrumented exit-code mismatch.
+ *
+ * @param {object} input confirmation inputs
+ * @param {object} input.command selected project command
+ * @param {object} input.framework manifest framework entry
+ * @param {object} input.options validator options
+ * @param {string} input.out validation output directory
+ * @param {string} input.scenarioName scenario identifier
+ * @returns {Promise<{artifacts: string[], evidence: object, result: object}>} clean confirmation outcome
+ */
+async function runCleanConfirmation ({ command, framework, options, out, scenarioName }) {
+  const outDir = frameworkOutDir(out, framework, `${scenarioName}-clean-confirmation`)
+  const result = await runCommand(getDatadogCleanCommand(command), {
+    artifactRoot: out,
+    envMode: 'clean',
+    label: `${framework.id}:${scenarioName}:clean-confirmation`,
+    outDir,
+    repositoryRoot: options.repositoryRoot,
+    requireExecutableApproval: options.requireExecutableApproval,
+    verbose: options.verbose,
+  })
+  const exitMatchesPreflight = !result.timedOut && matchesPreflightExitCode(framework.preflight, result.exitCode)
+  return {
+    artifacts: Object.values(result.artifacts),
+    result,
+    evidence: {
+      ran: true,
+      exitCode: result.exitCode,
+      timedOut: result.timedOut,
+      exitMatchesPreflight,
+      commandOutputSummary: summarizeTestOutput(result.stdout, result.stderr),
+    },
+  }
+}
+
+/**
+ * Describes an exit mismatch that cannot be attributed because clean executions disagreed.
+ *
+ * @param {object} preflight initial clean preflight evidence
+ * @param {object} instrumented instrumented command result
+ * @param {object} cleanConfirmation repeated clean command result
+ * @returns {string} customer-facing diagnosis
+ */
+function getUnstableBaselineDiagnosis (preflight, instrumented, cleanConfirmation) {
+  const confirmation = cleanConfirmation.timedOut
+    ? 'timed out'
+    : `exited ${cleanConfirmation.exitCode}`
+  return `The selected command exited ${preflight.exitCode} during the initial clean preflight, exited ` +
+    `${instrumented.exitCode} with Datadog initialized, and ${confirmation} during a second clean run. The ` +
+    'non-Datadog baseline was not stable, so no Test Optimization compatibility conclusion was reached.'
+}
+
+/**
+ * Describes an instrumented-only exit mismatch after the clean baseline was reproduced.
+ *
+ * @param {object} preflight initial clean preflight evidence
+ * @param {object} instrumented instrumented command result
+ * @returns {string} customer-facing diagnosis
+ */
+function getPossibleCompatibilityDiagnosis (preflight, instrumented) {
+  return `The selected command exited ${preflight.exitCode} in both runs without Datadog, but exited ` +
+    `${instrumented.exitCode} with Datadog initialized after reporting Test Optimization events. This may indicate ` +
+    'a dd-trace compatibility issue rather than a project baseline failure.'
+}
+
+function getBasicReportingCommand (framework) {
+  return getLocalValidationCommand(framework, framework.existingTestCommand)
+}
+
+async function failBasicReportingWithDebugRerun (options) {
+  const failure = await failWithDebugRerun(options)
+  return refineBasicReportingFailure(failure)
+}
+
+function refineBasicReportingFailure (failure) {
+  const evidence = failure.evidence || {}
+  if (['dd-trace-preload-failed', 'command-setup-failed'].includes(evidence.eventLevelFailure?.kind)) {
+    failure.status = 'error'
+    evidence.localDiagnosis = evidence.eventLevelFailure
+    return failure
+  }
+
+  const diagnosis = getDebugAwareDiagnosis(failure.diagnosis, evidence)
+  if (!diagnosis) return failure
+
+  failure.diagnosis = diagnosis.summary
+  evidence.localDiagnosis = diagnosis
+
+  if (evidence.eventLevelFailure) {
+    evidence.eventLevelFailure = {
+      ...evidence.eventLevelFailure,
+      summary: diagnosis.summary,
+      recommendation: diagnosis.recommendation,
+      signals: diagnosis.signals,
+    }
+  }
+
+  return failure
+}
+
+function getDebugAwareDiagnosis (currentDiagnosis, evidence) {
+  if (evidence.eventLevelFailure?.kind !== 'no-test-optimization-events') return null
+
+  const debugRerun = evidence.debugRerun
+  if (!debugRerun || debugRerun.ran !== true) return null
+
+  const testOutputSummary = summarizeTestOutput(
+    (evidence.commandOutputSummary || []).join('\n'),
+    (debugRerun.stdoutExcerpt || []).join('\n')
+  )
+  const testsRan = commandOutputShowsTestsRan(testOutputSummary)
+  const debugLine = findDebugLine(debugRerun, /dd-trace is not initialized in a package manager/i)
+  const noDebugEvents = !hasAnyTestOptimizationEvent(debugRerun)
+
+  if (testsRan && debugLine && noDebugEvents) {
+    return {
+      kind: 'tests-ran-tracer-not-initialized',
+      summary: 'The selected command ran tests, but no Test Optimization events reached the offline event artifact. ' +
+        `The debug rerun printed "${debugLine}", which means the preload executed in the package-manager ` +
+        'wrapper without producing Test Optimization events from the test process.',
+      recommendation: 'Try a direct test-runner command, or verify NODE_OPTIONS with dd-trace/ci/init reaches the ' +
+        'final test process rather than only the package-manager wrapper.',
+      signals: getDebugSignals({
+        debugLine,
+        debugRerun,
+        testOutputSummary,
+      }),
+    }
+  }
+
+  if (testsRan && noDebugEvents) {
+    return {
+      kind: 'tests-ran-no-test-optimization-events',
+      summary: 'The selected command ran tests, but no Test Optimization events reached the offline event artifact. ' +
+        'The debug rerun did not emit Test Optimization events either.',
+      recommendation: 'Inspect the debug rerun excerpt for tracer initialization or offline exporter errors, then ' +
+        'verify NODE_OPTIONS with dd-trace/ci/init reaches the final test process.',
+      signals: getDebugSignals({
+        debugRerun,
+        testOutputSummary,
+      }),
+    }
+  }
+
+  if (debugLine && noDebugEvents) {
+    return {
+      kind: 'tracer-not-initialized',
+      summary: `${currentDiagnosis} The debug rerun printed "${debugLine}".`,
+      recommendation: 'Verify NODE_OPTIONS with dd-trace/ci/init reaches the final test process.',
+      signals: getDebugSignals({
+        debugLine,
+        debugRerun,
+        testOutputSummary,
+      }),
+    }
+  }
+}
+
+function shouldRunDebugRerun (eventLevelFailure, result) {
+  return result.timedOut !== true &&
+    eventLevelFailure.kind !== 'vitest-benchmark' &&
+    eventLevelFailure.kind !== 'custom-jest-runner'
+}
+
+function matchesPreflightExitCode (preflight, exitCode) {
+  return preflight?.ran === true &&
+    Number.isInteger(preflight.exitCode) &&
+    preflight.exitCode === exitCode
+}
+
+function summarizePreflight (preflight) {
+  if (!preflight || preflight.ran !== true) {
+    return {
+      ran: false,
+      reason: 'No dd-trace-less preflight result was recorded in the manifest.',
+    }
+  }
+
+  return {
+    ran: true,
+    exitCode: preflight.exitCode,
+    observedTestCount: preflight.observedTestCount,
+    stdoutSummary: preflight.stdoutSummary,
+    stderrSummary: preflight.stderrSummary,
+  }
+}
+
+function summarizeTestOutput (stdout = '', stderr = '') {
+  return findInterestingLines(`${stdout}\n${stderr}`, [
+    /\b\d+\s+passing\b/i,
+    /\b\d+\s+pending\b/i,
+    /\b\d+\s+failing\b/i,
+    /\b\d+\s+passed\b/i,
+    /\b\d+\s+failed\b/i,
+    /\btests?\b.*\bpassed\b/i,
+    /\btests?\b.*\bfailed\b/i,
+    /\bSuccessfully ran target\b.*\btest\b/i,
+    /\bsuccess:\s*\d+\b/i,
+    /\bTasks:\s*\d+\s+successful\b/i,
+  ], 8)
+}
+
+function commandOutputShowsTestsRan (lines) {
+  return lines.some(line => {
+    return /\b\d+\s+(?:passing|passed)\b/i.test(line) ||
+      /\btests?\b.*\bpassed\b/i.test(line) ||
+      /\bSuccessfully ran target\b.*\btest\b/i.test(line) ||
+      /\bsuccess:\s*[1-9]\d*\b/i.test(line) ||
+      /\bfailed:\s*[1-9]\d*\b/i.test(line) ||
+      /\bTasks:\s*[1-9]\d*\s+successful\b/i.test(line)
+  })
+}
+
+function findDebugLine (debugRerun, pattern) {
+  const lines = [
+    ...(debugRerun.debugLines || []),
+    ...(debugRerun.stdoutExcerpt || []),
+    ...(debugRerun.stderrExcerpt || []),
+  ]
+  return lines.find(line => pattern.test(line))
+}
+
+function getDebugSignals ({ debugLine, debugRerun, testOutputSummary }) {
+  return {
+    debugLine,
+    debugLines: debugRerun.debugLines || [],
+    stdoutExcerpt: debugRerun.stdoutExcerpt || [],
+    stderrExcerpt: debugRerun.stderrExcerpt || [],
+    testOutputSummary,
+  }
+}
+
+function summarizeCommandFailure (result, evidence) {
+  const output = `${result.stdout}\n${result.stderr}`
+  const buildErrors = findInterestingLines(output, [
+    /Could not resolve /,
+    /Cannot find module/,
+    /Module not found/,
+    /Error \[ERR_MODULE_NOT_FOUND\]/,
+  ])
+  const assertionFailures = findInterestingLines(output, [
+    /AssertionError/,
+    /Timed out retrying/,
+    /^\s+\d+\) /,
+  ])
+  const testEventsWereReported = evidence.testSessionEvents > 0 &&
+    evidence.testModuleEvents > 0 &&
+    evidence.testSuiteEvents > 0 &&
+    evidence.testEvents > 0
+  const summary = getFailureSummary({
+    buildErrors,
+    assertionFailures,
+    exitCode: result.exitCode,
+    testEventsWereReported,
+    timedOut: result.timedOut,
+  })
+
+  return {
+    assertionFailures,
+    buildErrors,
+    stderrExcerpt: tailInterestingLines(result.stderr),
+    stdoutExcerpt: tailInterestingLines(result.stdout),
+    summary,
+    testEventsWereReported,
+  }
+}
+
+function getMissingEventDiagnosis ({ framework, result, evidence }) {
+  const missingLevels = getMissingLevels(evidence)
+  const commandFailure = evidence.commandFailure
+  const vitestBenchmark = detectVitestBenchmark(framework, result)
+  const frameworkSourceTreeRunner = detectFrameworkSourceTreeRunner(framework, result)
+  const customJestRunner = detectCustomJestRunner(framework)
+
+  if (commandFailure?.buildErrors?.length > 0 && !hasAnyTestOptimizationEvent(evidence)) {
+    const preloadFailed = /(?:ci\/init\.js|internal\/preload)/.test(`${result.stdout}\n${result.stderr}`)
+    return {
+      kind: preloadFailed ? 'dd-trace-preload-failed' : 'command-setup-failed',
+      missingLevels,
+      signals: commandFailure.buildErrors,
+      summary: preloadFailed
+        ? `The Test Optimization preload failed before tests started: ${commandFailure.buildErrors[0]}. ` +
+          'No Test Optimization conclusion was reached.'
+        : `${commandFailure.summary} No Test Optimization conclusion was reached.`,
+      recommendation: preloadFailed
+        ? 'Repair or reinstall dd-trace so its declared dependencies resolve, then rerun validation.'
+        : 'Fix the selected project command or its setup, then rerun validation.',
+    }
+  }
+
+  if (vitestBenchmark) {
+    return {
+      kind: 'vitest-benchmark',
+      missingLevels,
+      signals: vitestBenchmark.signals,
+      summary: 'The selected Vitest command appears to run benchmark mode, not normal tests. ' +
+        'Test Optimization reported session/module/suite events, but Vitest benchmark mode did not emit ' +
+        'per-test events. Choose a normal Vitest test command such as "vitest run <test-file>" for validation.',
+      recommendation: 'Replace the selected command with a normal Vitest test command; do not use `vitest bench` ' +
+        'or benchmark-only `*.bench.*` files for Test Optimization validation.',
+    }
+  }
+
+  if (frameworkSourceTreeRunner && !hasAnyTestOptimizationEvent(evidence)) {
+    return {
+      kind: 'framework-source-tree-runner',
+      missingLevels,
+      signals: frameworkSourceTreeRunner.signals,
+      summary: 'The selected command ran tests from the test framework source tree, but no Test Optimization ' +
+        'events reached the offline event artifact. This command is not equivalent to a customer project running an ' +
+        'installed test-runner package.',
+      recommendation: 'Choose a project test command that uses an installed supported framework package. If this ' +
+        'repository is the framework itself, treat this entry as not runnable for Test Optimization validation.',
+    }
+  }
+
+  if (!hasAnyTestOptimizationEvent(evidence)) {
+    return {
+      kind: 'no-test-optimization-events',
+      missingLevels,
+      summary: 'No Test Optimization test events reached the offline event artifact. The tracer may not have ' +
+        'initialized in the test process, or the selected command may not have executed tests.',
+      recommendation: 'Check the debug rerun output for tracer initialization or offline exporter errors.',
+    }
+  }
+
+  if (evidence.testEvents === 0) {
+    if (customJestRunner) {
+      return {
+        kind: 'custom-jest-runner',
+        missingLevels,
+        customTestRunner: customJestRunner,
+        signals: customJestRunner.signals,
+        summary: `We detected a custom Jest-compatible runner: \`${customJestRunner.name}\`. The test command ` +
+          'executes tests, but it does not use a currently supported Jest entrypoint. Test Optimization ' +
+          'initialized, but this runner did not emit the Jest lifecycle events needed to report individual ' +
+          'suites and tests.',
+        recommendation: 'Try a standard Jest runner command for validation, or choose a test command that does not ' +
+          'use the custom runner. If this project must use the custom runner, dd-trace may need explicit support ' +
+          'for that runner before per-test reporting and advanced Test Optimization features can work.',
+      }
+    }
+
+    return {
+      kind: 'missing-test-events',
+      missingLevels,
+      summary: 'Test Optimization initialized and emitted higher-level events, but per-test events were missing. ' +
+        'This usually points to an unsupported runner mode, unsupported framework configuration, or per-test hooks ' +
+        'not firing for the selected command.',
+      recommendation: 'Choose a smaller standard test command, then inspect the debug rerun output for hook or ' +
+        'exporter errors.',
+    }
+  }
+
+  return {
+    kind: 'missing-event-levels',
+    missingLevels,
+    summary: `The command ran, but these required Test Optimization event levels were missing: ${
+      missingLevels.join(', ')
+    }.`,
+    recommendation: 'Inspect the debug rerun output for tracer initialization, hook, or exporter errors.',
+  }
+}
+
+function getMissingLevels (evidence) {
+  const missing = []
+  if (evidence.testSessionEvents === 0) missing.push('test_session_end')
+  if (evidence.testModuleEvents === 0) missing.push('test_module_end')
+  if (evidence.testSuiteEvents === 0) missing.push('test_suite_end')
+  if (evidence.testEvents === 0) missing.push('test')
+  return missing
+}
+
+function hasAnyTestOptimizationEvent (evidence) {
+  return evidence.testSessionEvents > 0 ||
+    evidence.testModuleEvents > 0 ||
+    evidence.testSuiteEvents > 0 ||
+    evidence.testEvents > 0
+}
+
+function detectVitestBenchmark (framework, result) {
+  if (framework.framework !== 'vitest') return null
+
+  const command = result.command || ''
+  const output = `${result.stdout}\n${result.stderr}`
+  const signals = []
+
+  if (/\bvitest\s+bench\b/.test(command)) signals.push('command contains `vitest bench`')
+  if (/\.bench\.[cm]?[jt]sx?\b/.test(command)) signals.push('command targets a `*.bench.*` file')
+  if (/^\s*BENCH\s+Summary\b/m.test(output)) signals.push('stdout contains a Vitest BENCH summary')
+  if (/Benchmarking is an experimental feature/.test(output)) {
+    signals.push('stderr says Vitest benchmarking is experimental')
+  }
+
+  return signals.length > 0 ? { signals } : null
+}
+
+function detectCustomJestRunner (framework) {
+  if (framework.framework !== 'jest') return null
+
+  const configRunner = findJestRunnerInConfigFiles(framework.project?.configFiles || [])
+  if (configRunner && isCustomJestRunner(configRunner.name)) return configRunner
+
+  const packageRunner = findJestRunnerInPackageJson(framework.project?.packageJson)
+  if (packageRunner && isCustomJestRunner(packageRunner.name)) return packageRunner
+}
+
+function findJestRunnerInConfigFiles (configFiles) {
+  for (const configFile of configFiles) {
+    const content = readFile(configFile)
+    if (!content) continue
+
+    const match = /(?:^|[,{]\s*)runner\s*:\s*['"]([^'"]+)['"]/.exec(content)
+    if (!match) continue
+
+    return {
+      name: match[1],
+      source: configFile,
+      sourceType: 'config',
+      signals: [
+        `Jest config ${configFile} sets runner: ${match[1]}`,
+      ],
+    }
+  }
+}
+
+function findJestRunnerInPackageJson (packageJsonPath) {
+  if (!packageJsonPath) return
+
+  try {
+    const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'))
+    const runner = packageJson.jest?.runner
+    if (typeof runner !== 'string' || runner.length === 0) return
+
+    return {
+      name: runner,
+      source: packageJsonPath,
+      sourceType: 'package.json',
+      signals: [
+        `package.json jest.runner is ${runner}`,
+      ],
+    }
+  } catch {}
+}
+
+function isCustomJestRunner (runner) {
+  return runner !== 'jest-runner'
+}
+
+function detectFrameworkSourceTreeRunner (framework, result) {
+  if (framework.framework !== 'mocha') return null
+
+  const projectName = framework.project?.name || ''
+  const commandAndOutput = [
+    result.command || '',
+    result.stdout || '',
+    result.stderr || '',
+  ].join('\n')
+  const signals = []
+
+  if (projectName === 'mocha') {
+    signals.push('repository package name is `mocha`')
+  }
+  if (/\bnode\s+\.\/bin\/mocha\.js\b/.test(commandAndOutput) ||
+    /\bnode\s+bin\/mocha\.js\b/.test(commandAndOutput)) {
+    signals.push('selected command invokes the repository-local `bin/mocha.js` source-tree runner')
+  }
+  if (fileExists(framework.project?.root, 'lib/mocha.cjs') && fileExists(framework.project?.root, 'lib/runner.cjs')) {
+    signals.push('repository contains Mocha source files `lib/mocha.cjs` and `lib/runner.cjs`')
+  }
+
+  return signals.length >= 2 ? { signals } : null
+}
+
+function fileExists (root, filename) {
+  if (!root) return false
+
+  try {
+    return fs.existsSync(path.join(root, filename))
+  } catch {
+    return false
+  }
+}
+
+function readFile (filename) {
+  try {
+    return fs.readFileSync(filename, 'utf8')
+  } catch {}
+}
+
+function getFailureSummary ({ buildErrors, assertionFailures, exitCode, testEventsWereReported, timedOut }) {
+  if (timedOut) {
+    return 'The selected test command timed out before payload validation could pass.'
+  }
+
+  if (buildErrors.length > 0) {
+    if (testEventsWereReported) {
+      return 'The selected test command reported Datadog test events, but project setup/build errors made it fail.'
+    }
+
+    return 'The selected test command failed during project setup/build before payload validation could pass.'
+  }
+
+  if (assertionFailures.length > 0 && testEventsWereReported) {
+    return 'The selected test command reported Datadog test events, but the tests failed.'
+  }
+
+  if (testEventsWereReported) {
+    return `The selected test command reported Datadog test events, but exited ${exitCode}.`
+  }
+
+  return `The selected test command exited ${exitCode} before payload validation could pass.`
+}
+
+module.exports = {
+  getDebugAwareDiagnosis,
+  getBasicReportingCommand,
+  getMissingEventDiagnosis,
+  refineBasicReportingFailure,
+  runBasicReporting,
+  shouldRunDebugRerun,
+  summarizeTestOutput,
+}