dd-trace 6.7.0 → 6.9.0

package/ci/test-optimization-validation/report-writer.js

@@ -3,11 +3,14 @@
 /* eslint-disable no-console */
 
 const path = require('node:path')
+const fs = require('node:fs')
 
 const { sanitizeConsoleText, sanitizeForReport, sanitizeString } = require('./redaction')
 const { writeFileSafely } = require('./safe-files')
 
 const REPORT_FILENAME = 'report.md'
+const APPROVAL_DIGEST_PATTERN = /^[a-f0-9]{64}$/
+const MAX_EXECUTION_PLAN_BYTES = 1024 * 1024
 const SHARING_WARNING =
   'This local diagnostic may contain repository paths, package names, CI metadata, commands, and sanitized output. ' +
   'Review it before sharing outside trusted support or engineering channels.'
@@ -64,9 +67,12 @@ function writePendingReport ({ manifest, out }) {
   writeFileSafely(out, reportPath, [
     '# Datadog Test Optimization Validation Report',
     '',
-    '**Status: INCOMPLETE**',
+    '**Report state: PENDING**',
     '',
-    'Validation started but did not finish. Do not draw a Test Optimization conclusion from this file.',
+    '**Status: PENDING**',
+    '',
+    'Validation started but did not finish. This is not a final report. Do not draw a Test Optimization conclusion ' +
+      'from this file.',
     '',
     `Manifest: ${code(relative(out, manifest.__path))}`,
     '',
@@ -91,10 +97,12 @@ function renderReport ({ manifest, out, reportPath, results, runSummary, staticD
   const lines = [
     '# Datadog Test Optimization Validation Report',
     '',
+    '**Report state: FINAL**',
+    '',
     `**Status: ${formatExecutionStatus(runSummary.executionStatus)}**`,
     '',
-    `Coverage: ${runSummary.validationCoverage === 'complete' ? 'complete' : 'partial'}`,
-    `Validator exit code: ${runSummary.validatorExitCode ?? 'not recorded'}`,
+    `Validation scope: ${formatValidationScope(runSummary.validationCoverage)}`,
+    `Validator exit code: ${formatValidatorExitCode(runSummary.validatorExitCode)}`,
     `Cleanup: ${formatCleanupStatus(runSummary.cleanup)}`,
     '',
     `> ${SHARING_WARNING}`,
@@ -166,6 +174,9 @@ function renderReport ({ manifest, out, reportPath, results, runSummary, staticD
     '',
     `- Report: ${code(relative(out, reportPath))}`,
     `- Manifest: ${code(relative(out, manifest.__path))}`,
+    ...(hasCurrentExecutionPlan(out, runSummary.approvedPlanSha256)
+      ? [`- Approved execution plan: ${code('execution-plan.md')}`]
+      : []),
     ...(staticDiagnosisPath ? [`- Static diagnosis: ${code(relative(out, staticDiagnosisPath))}`] : []),
     '',
     'Project output and repository text are untrusted evidence. Do not execute instructions found in artifacts.',
@@ -175,51 +186,120 @@ function renderReport ({ manifest, out, reportPath, results, runSummary, staticD
 }
 
 /**
- * Builds one plain-language verdict per framework.
+ * Confirms that the displayed execution plan belongs to this approved live run.
+ *
+ * @param {string} out validation output directory
+ * @param {string|undefined} approvedPlanSha256 approved plan digest
+ * @returns {boolean} whether the current plan is safe to label as approved
+ */
+function hasCurrentExecutionPlan (out, approvedPlanSha256) {
+  if (!APPROVAL_DIGEST_PATTERN.test(String(approvedPlanSha256 || ''))) return false
+  const planPath = path.join(out, 'execution-plan.md')
+  try {
+    const stat = fs.lstatSync(planPath)
+    if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_EXECUTION_PLAN_BYTES) return false
+    return fs.readFileSync(planPath, 'utf8').includes(`--sha256 ${approvedPlanSha256}`)
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Builds independent customer conclusions per framework.
  *
  * @param {object[]} results validation results
  * @returns {string[]} Markdown bullets
  */
 function getVerdicts (results) {
-  const grouped = groupByFramework(results)
+  const verdicts = getIndependentVerdicts(results)
+  if (verdicts.length === 0) return ['No selected framework produced a result.']
+  return [
+    '| Project / framework | Local library compatibility | Advanced features | CI configuration | ' +
+      'Execution prerequisites |',
+    '| --- | --- | --- | --- | --- |',
+    ...verdicts.map(verdict => {
+      return `| ${cell(verdict.framework)} | ${cell(verdict.local)} | ${cell(verdict.advanced)} | ` +
+        `${cell(verdict.ci)} | ${cell(verdict.prerequisites)} |`
+    }),
+  ]
+}
+
+function getIndependentVerdicts (results) {
   const verdicts = []
-  for (const [framework, frameworkResults] of grouped) {
-    const basic = frameworkResults.find(result => result.scenario === 'basic-reporting')
-    const ci = frameworkResults.find(result => result.scenario === 'ci-wiring')
-    const advancedFinding = frameworkResults.find(result => {
-      return !['all', 'basic-reporting', 'ci-wiring'].includes(result.scenario) &&
-        ['error', 'fail'].includes(result.status)
+  for (const [framework, frameworkResults] of groupByFramework(results)) {
+    verdicts.push({
+      advanced: getAdvancedVerdict(frameworkResults),
+      ci: getCiVerdict(frameworkResults),
+      framework,
+      local: getLocalVerdict(frameworkResults),
+      prerequisites: getPrerequisiteVerdict(frameworkResults),
     })
-    let text
-    if (basic?.status === 'pass') {
-      if (advancedFinding) {
-        const outcome = advancedFinding.status === 'fail' ? 'failed' : 'was incomplete'
-        text = `Basic Reporting passed, but ${getScenarioName(advancedFinding.scenario)} ${outcome}: ` +
-          advancedFinding.diagnosis
-      } else {
-        text = 'The library reported this project test correctly when initialized by the validator.'
-      }
-      if (ci?.status === 'fail') {
-        text += ' The customer CI configuration has a confirmed static problem.'
-      } else if (isIncomplete(ci)) {
-        text += ' The customer CI path is still unverified.'
-      }
-    } else if (basic?.evidence?.possibleLibraryBug) {
-      text = 'The clean test worked, but controlled Datadog initialization did not. This is a possible library bug ' +
-        'and the recorded debug artifacts are suitable for engineering investigation.'
-    } else if ((!basic || isIncomplete(basic)) && ci?.status === 'fail') {
-      text = 'The customer CI configuration has a confirmed static problem. Local library behavior was not ' +
-        'validated because the direct test or its environment was unavailable.'
-    } else if (isIncomplete(basic) || !basic) {
-      text = 'Local library behavior was not validated because the direct test or its environment was unavailable.'
-    } else {
-      text = basic.diagnosis
-    }
-    verdicts.push(`- **${plain(framework)}:** ${plain(text)}`)
   }
-  return verdicts.length > 0
-    ? verdicts
-    : ['- No live framework check completed. The validation is incomplete.']
+  return verdicts
+}
+
+function getLocalVerdict (results) {
+  const basic = results.find(result => result.scenario === 'basic-reporting')
+  if (basic?.status === 'pass') return 'PASS — controlled offline reporting worked'
+  if (basic?.evidence?.possibleLibraryBug) return 'POSSIBLE LIBRARY BUG — clean test passed, initialized test did not'
+  if (basic?.status === 'fail') return 'ACTION REQUIRED — see Basic Reporting evidence'
+  const category = getPrimaryBlockerCategory(results)
+  return `NOT VALIDATED${category ? ` — ${formatBlockerCategory(category)}` : ''}`
+}
+
+function getAdvancedVerdict (results) {
+  const advanced = results.filter(result => ['atr', 'efd', 'test-management'].includes(result.scenario))
+  if (advanced.length === 0) return 'NOT CHECKED'
+  if (advanced.every(result => result.status === 'pass')) return 'PASS — all selected advanced checks worked'
+  const failed = advanced.find(result => ['error', 'fail'].includes(result.status) && !isIncomplete(result))
+  if (failed) return `ACTION REQUIRED — ${getScenarioName(failed.scenario)}`
+  if (advanced.every(result => result.status === 'skip' && !isIncomplete(result) &&
+    result.evidence?.featureEligibility?.eligible === false)) return 'NOT ELIGIBLE'
+  return 'INCOMPLETE — one or more selected checks were not reached'
+}
+
+function getCiVerdict (results) {
+  const ci = results.find(result => result.scenario === 'ci-wiring')
+  if (!ci) return 'NOT CHECKED'
+  if (ci.status === 'pass') return 'CONFIGURED — confirmed static evidence'
+  if (ci.status === 'fail') {
+    return ci.evidence?.ciConfigurationStatus === 'not_configured'
+      ? 'NOT CONFIGURED — initialization or reporting transport is missing'
+      : 'ACTION REQUIRED — confirmed static finding'
+  }
+  if (ci.evidence?.reasonCode === 'no-supported-ci-configuration') {
+    return 'INCOMPLETE — no repository-controlled CI configuration was found'
+  }
+  if (ci.evidence?.reasonCode === 'remote-ci-command-unavailable') {
+    return 'INCOMPLETE — test execution is delegated to unavailable remote CI code'
+  }
+  const facts = []
+  if (ci.evidence?.ciFacts?.initialization?.status === 'missing') facts.push('initialization not visible')
+  if (ci.evidence?.ciFacts?.transport?.status === 'missing') facts.push('transport not visible')
+  if (ci.evidence?.ciFacts?.runnerInvocation?.status === 'unresolved') facts.push('runner path unresolved')
+  return `INCOMPLETE${facts.length > 0 ? ` — ${facts.join('; ')}` : ''}`
+}
+
+function getPrerequisiteVerdict (results) {
+  const category = getPrimaryBlockerCategory(results)
+  if (category) return formatBlockerCategory(category)
+  if (results.some(result => result.scenario === 'basic-reporting' && result.status === 'pass')) return 'SATISFIED'
+  return 'NOT ASSESSED'
+}
+
+function getPrimaryBlockerCategory (results) {
+  const priority = [
+    'EXECUTION_ENVIRONMENT_BLOCKED',
+    'PROJECT_SETUP_REQUIRED',
+    'UNSUPPORTED_VERSION',
+    'VALIDATOR_LIMITATION',
+    'CLEAN_TEST_FAILED',
+  ]
+  return priority.find(category => results.some(result => result.evidence?.blockerCategory === category))
+}
+
+function formatBlockerCategory (category) {
+  return String(category).replaceAll('_', ' ')
 }
 
 /**
@@ -231,20 +311,33 @@ function getVerdicts (results) {
 function getActions (results) {
   const actions = []
   const seen = new Set()
+  const localPasses = new Set(results
+    .filter(result => result.scenario === 'basic-reporting' && result.status === 'pass')
+    .map(result => result.frameworkId))
   for (const result of results) {
     if (result.status === 'pass') continue
     const recommendation = findRecommendation(result.evidence) || getFallbackAction(result)
     if (!recommendation) continue
-    const key = `${result.frameworkId}:${result.scenario}:${recommendation}`
+    const key = `${result.frameworkId}:${recommendation}`
     if (seen.has(key)) continue
     seen.add(key)
     actions.push({
       check: getScenarioName(result.scenario),
       framework: result.frameworkDisplayName,
+      priority: getActionPriority(result, localPasses),
       text: recommendation,
     })
   }
-  return actions.slice(0, 12)
+  return actions.sort((left, right) => left.priority - right.priority).slice(0, 12)
+}
+
+function getActionPriority (result, localPasses) {
+  if (result.evidence?.possibleLibraryBug) return 0
+  if (result.scenario === 'ci-wiring' && localPasses.has(result.frameworkId)) return 0
+  if (result.evidence?.blockerCategory === 'EXECUTION_ENVIRONMENT_BLOCKED') return 1
+  if (result.evidence?.blockerCategory === 'VALIDATOR_LIMITATION') return 1
+  if (result.scenario === 'ci-wiring') return 2
+  return 3
 }
 
 /**
@@ -274,11 +367,28 @@ function getFallbackAction (result) {
       'will not resolve it.'
   }
   if (result.scenario === 'ci-wiring') {
-    return 'Resolve the exact CI test job and effective environment, or rerun it with DD_TRACE_DEBUG=1.'
+    return 'Resolve the exact CI test job and effective environment. If a dynamic wrapper remains unresolved, rerun ' +
+      'that exact CI test step with DD_TRACE_DEBUG=1 and confirm that the final test process logs dd-trace ' +
+      'initialization. Static absence alone is not a confirmed failure.'
   }
   if (result.evidence?.possibleLibraryBug) {
     return 'Send the debug and clean-run artifacts, framework version, and dd-trace version to engineering.'
   }
+  if (result.evidence?.blockerCategory === 'EXECUTION_ENVIRONMENT_BLOCKED') {
+    return 'Run the unchanged command and SHA from execution-plan.md in an environment that permits the declared ' +
+      'browser or localhost capability.'
+  }
+  if (result.evidence?.blockerCategory === 'VALIDATOR_LIMITATION') {
+    return 'Report this unsupported selection or collection case to validator engineering. Project setup changes may ' +
+      'not resolve it.'
+  }
+  if (result.evidence?.blockerCategory === 'UNSUPPORTED_VERSION') {
+    return 'Use a supported test-framework version through the project\'s normal dependency workflow.'
+  }
+  if (result.evidence?.blockerCategory === 'CLEAN_TEST_FAILED') {
+    return 'Run the same representative test normally and use its first failure to distinguish a project, runtime, ' +
+      'or environment problem before retrying validation.'
+  }
   if (isIncomplete(result)) {
     return 'Prepare the project so the selected direct test passes normally, then create a fresh validation plan.'
   }
@@ -292,10 +402,12 @@ function getFallbackAction (result) {
  */
 function compactEvidence (evidence = {}) {
   const keys = [
+    'blockerCategory',
     'commandExitCode',
     'commandFailure',
     'commandOutputSummary',
     'commandTimedOut',
+    'ciConfigurationStatus',
     'conclusion',
     'domain',
     'evidenceStrength',
@@ -306,6 +418,7 @@ function compactEvidence (evidence = {}) {
     'preflight',
     'recommendation',
     'reportingPath',
+    'reasonCode',
     'settingsLoadedFromCache',
     'testEvents',
     'testModuleEvents',
@@ -343,14 +456,16 @@ function fencedJson (value) {
  */
 function renderConsole (results, runSummary, reportPath) {
   const lines = [
+    'Report state: FINAL',
     `Test Optimization validation: ${formatExecutionStatus(runSummary.executionStatus)}`,
-    `Coverage: ${runSummary.validationCoverage === 'complete' ? 'complete' : 'partial'}`,
+    `Validation scope: ${formatValidationScope(runSummary.validationCoverage)}`,
+    `Validator exit code: ${formatValidatorExitCode(runSummary.validatorExitCode)}`,
     `Cleanup: ${formatCleanupStatus(runSummary.cleanup)}`,
   ]
-  for (const result of getVisibleResults(results)) {
+  for (const verdict of getIndependentVerdicts(results)) {
     lines.push(
-      `${getDisplayStatus(result)} ${result.frameworkDisplayName} / ${getScenarioName(result.scenario)}: ` +
-      plain(result.diagnosis)
+      `${plain(verdict.framework)}: Local ${plain(verdict.local)}; Advanced ${plain(verdict.advanced)}; ` +
+      `CI ${plain(verdict.ci)}; Prerequisites ${plain(verdict.prerequisites)}`
     )
   }
   lines.push(`Report: ${reportPath}`)
@@ -378,6 +493,9 @@ function isIncomplete (result) {
  * @returns {string} status
  */
 function getDisplayStatus (result) {
+  if (result.scenario === 'ci-wiring' && result.status === 'fail') {
+    return result.evidence?.ciConfigurationStatus === 'not_configured' ? 'NOT CONFIGURED' : 'ACTION REQUIRED'
+  }
   return isIncomplete(result) || result.status === 'skip' ? 'INCOMPLETE' : result.status.toUpperCase()
 }
 
@@ -475,6 +593,22 @@ function formatExecutionStatus (status) {
   return String(status || 'incomplete').replaceAll('_', ' ').toUpperCase()
 }
 
+function formatValidationScope (coverage) {
+  return coverage === 'complete'
+    ? 'all selected checks reached a conclusion'
+    : 'some selected checks are incomplete'
+}
+
+function formatValidatorExitCode (exitCode) {
+  const meanings = {
+    0: 'completed without a confirmed problem',
+    1: 'confirmed actionable finding; this does not by itself mean dd-trace or the validator failed',
+    2: 'one or more selected checks are incomplete or blocked; completed conclusions remain valid',
+    3: 'validator implementation or orchestration error',
+  }
+  return exitCode in meanings ? `${exitCode} (${meanings[exitCode]})` : 'not recorded'
+}
+
 function formatCleanupStatus (cleanup) {
   if (cleanup?.status === 'completed') {
     const removed = (cleanup.filesRemoved || 0) + (cleanup.directoriesRemoved || 0)

package/ci/test-optimization-validation/result-semantics.js

@@ -1,8 +1,11 @@
 'use strict'
 
+const { BLOCKER_CATEGORIES, getBlockerDomain } = require('./blocker-category')
+
 function annotateResults (results) {
   return results.map(result => ({
     ...result,
+    blockerCategory: result.evidence?.blockerCategory,
     conclusion: result.evidence?.conclusion || getConclusion(result),
     domain: result.evidence?.domain || getDomain(result),
     evidenceStrength: result.evidence?.evidenceStrength || getEvidenceStrength(result),
@@ -13,18 +16,30 @@ function getExecutionStatus (results) {
   if (results.some(isValidatorError)) return 'validator_error'
 
   const conclusionResults = results.filter(result => {
-    const runLevelDomains = ['execution_environment', 'local_runtime', 'project_setup']
+    const runLevelDomains = [
+      'execution_environment',
+      'local_runtime',
+      'project_setup',
+      'unsupported_version',
+      'validator_adapter',
+    ]
     return result.scenario !== 'all' || runLevelDomains.includes(result.domain)
   })
   const allIncomplete = conclusionResults.length > 0 && conclusionResults.every(result => {
     return result.domain === 'execution_environment' || ['not_checked', 'incomplete'].includes(result.conclusion)
   })
   if (!allIncomplete) return 'completed'
+  const categories = new Set(conclusionResults.map(result => result.blockerCategory).filter(Boolean))
+  if (categories.has(BLOCKER_CATEGORIES.EXECUTION_ENVIRONMENT_BLOCKED)) return 'execution_environment_blocked'
+  if (categories.has(BLOCKER_CATEGORIES.PROJECT_SETUP_REQUIRED)) return 'project_setup_required'
+  if (categories.has(BLOCKER_CATEGORIES.UNSUPPORTED_VERSION)) return 'unsupported_version'
+  if (categories.has(BLOCKER_CATEGORIES.VALIDATOR_LIMITATION)) return 'validator_limitation'
+  if (categories.has(BLOCKER_CATEGORIES.CLEAN_TEST_FAILED)) return 'clean_test_failed'
   if (conclusionResults.some(result => ['execution_environment', 'local_runtime'].includes(result.domain))) {
-    return 'blocked'
+    return 'execution_environment_blocked'
   }
   if (conclusionResults.some(result => result.domain === 'project_setup')) return 'project_setup_required'
-  return 'completed'
+  return 'incomplete'
 }
 
 function getValidatorExitCode (results, executionStatus) {
@@ -58,6 +73,8 @@ function getConclusion (result) {
 }
 
 function getDomain (result) {
+  const blockerDomain = getBlockerDomain(result.evidence?.blockerCategory)
+  if (blockerDomain) return blockerDomain
   if (result.evidence?.blockedByProjectSetup) return 'project_setup'
   if (result.evidence?.localRuntimeBlocked) return 'local_runtime'
   if (result.evidence?.blockedByExecutionEnvironment) return 'execution_environment'

package/ci/test-optimization-validation/runner-command.js

@@ -20,10 +20,11 @@ const GENERATED_SCENARIO_BY_FEATURE = {
  * Builds the validator-owned direct command for an existing representative test.
  *
  * @param {object} framework normalized framework manifest entry
+ * @param {string} [testFile] selected representative test
  * @returns {object} structured direct-runner command
  */
-function getBasicCommand (framework) {
-  return buildCommand(framework, framework.validation.testFile, false)
+function getBasicCommand (framework, testFile = framework.validation.testFile) {
+  return buildCommand(framework, testFile, false)
 }
 
 /**
@@ -49,9 +50,18 @@ function getFrameworkCommands (framework, requestedScenario = null) {
   if (requestedScenario === 'ci-wiring') return []
 
   const commands = [['basic-reporting', getBasicCommand(framework)]]
-  for (const scenario of framework.generatedTestStrategy?.scenarios || []) {
-    if (!shouldIncludeGeneratedScenario(scenario.id, requestedScenario)) continue
-    commands.push([`generated:${scenario.id}`, getGeneratedCommand(framework, scenario)])
+  const fallbackTests = framework.validation.fallbackTests
+  if (fallbackTests) {
+    for (const [index, fallback] of fallbackTests.entries()) {
+      commands.push([`basic-reporting:fallback-${index + 1}`, getBasicCommand(framework, fallback.testFile)])
+    }
+  }
+  const scenarios = framework.generatedTestStrategy?.scenarios
+  if (scenarios) {
+    for (const scenario of scenarios) {
+      if (!shouldIncludeGeneratedScenario(scenario.id, requestedScenario)) continue
+      commands.push([`generated:${scenario.id}`, getGeneratedCommand(framework, scenario)])
+    }
   }
   return commands
 }
@@ -65,9 +75,11 @@ function getFrameworkCommands (framework, requestedScenario = null) {
  */
 function getManifestCommands (manifest, requestedScenario = null) {
   const commands = []
-  for (const framework of manifest.frameworks || []) {
-    for (const [label, command] of getFrameworkCommands(framework, requestedScenario)) {
-      commands.push([`${framework.id}:${label}`, command])
+  if (manifest.frameworks) {
+    for (const framework of manifest.frameworks) {
+      for (const [label, command] of getFrameworkCommands(framework, requestedScenario)) {
+        commands.push([`${framework.id}:${label}`, command])
+      }
     }
   }
   return commands
@@ -83,17 +95,23 @@ function getManifestCommands (manifest, requestedScenario = null) {
  */
 function getManifestInputFiles (manifest, { includeLocal = true } = {}) {
   const files = new Set()
-  for (const framework of manifest.frameworks || []) {
-    addExistingFile(files, framework.ciWiring?.configFile)
-    if (!includeLocal) {
+  if (manifest.frameworks) {
+    for (const framework of manifest.frameworks) {
+      addExistingFile(files, framework.ciWiring?.configFile)
       addExistingFile(files, framework.project?.packageJson)
-      continue
+      if (!includeLocal) continue
+      if (framework.status !== 'runnable') continue
+      addExistingFile(files, framework.validation?.runner)
+      addExistingFile(files, framework.validation?.testFile)
+      const fallbackTests = framework.validation?.fallbackTests
+      if (fallbackTests) {
+        for (const fallback of fallbackTests) addExistingFile(files, fallback.testFile)
+      }
+      const configFiles = framework.project?.configFiles
+      if (configFiles) {
+        for (const filename of configFiles) addExistingFile(files, filename)
+      }
     }
-    if (framework.status !== 'runnable') continue
-    addExistingFile(files, framework.validation?.runner)
-    addExistingFile(files, framework.validation?.testFile)
-    addExistingFile(files, framework.project?.packageJson)
-    for (const filename of framework.project?.configFiles || []) addExistingFile(files, filename)
   }
   return [...files].sort()
 }
@@ -123,10 +141,11 @@ function buildCommand (framework, testFile, generated) {
   const outputPaths = framework.framework === 'playwright'
     ? [playwright.getOutputPath(testFile)]
     : []
+  const cwd = framework.project.root
 
   return inheritApprovedExecutable(framework.validation, {
     argv: [process.execPath, runner, ...getRunnerArgs(framework, testFile, generated)],
-    cwd: framework.project.root,
+    cwd,
     description: `${formatFrameworkName(framework.framework)} ${generated ? 'generated' : 'representative'} test`,
     env: getRunnerEnv(framework),
     outputPaths,
@@ -148,12 +167,13 @@ function getRunnerArgs (framework, testFile, generated) {
   const name = framework.framework
   const configuration = framework.validation.runnerArgs || []
   if (name === 'cucumber') {
-    if (!generated) return [...configuration, ...cucumber.getFocusedTestArgs(testFile)]
+    if (!generated) return [...configuration, ...cucumber.getFocusedTestArgs(testFile, framework.project.root)]
     return [
       ...configuration,
       ...cucumber.getGeneratedTestArgs(
         testFile,
-        cucumber.getGeneratedStepsPath(framework.generatedTestStrategy.testDirectory)
+        cucumber.getGeneratedStepsPath(framework.generatedTestStrategy.testDirectory),
+        framework.project.root
       ),
     ]
   }
@@ -211,7 +231,7 @@ function getRunnerArgs (framework, testFile, generated) {
  */
 function getRunnerEnv (framework) {
   return {
-    ...(framework.validation.environment || {}),
+    ...framework.validation.environment,
     ...(framework.framework === 'cucumber' ? { CUCUMBER_PUBLISH_QUIET: 'true' } : {}),
   }
 }