dd-trace 6.4.0 → 6.5.0

package/ci/test-optimization-validation/cli.js

@@ -0,0 +1,903 @@
+'use strict'
+
+/* eslint-disable no-console */
+
+const fs = require('fs')
+const path = require('path')
+
+const { getFrameworkDefinitions } = require('../diagnose')
+const { DD_MAJOR } = require('../../version')
+
+const { assertApprovalDigest, getApprovalDigest } = require('./approval')
+const { loadApprovedPlan } = require('./approval-artifacts')
+
+const { runBasicReporting } = require('./scenarios/basic-reporting')
+const { runEarlyFlakeDetection } = require('./scenarios/early-flake-detection')
+const { runAutoTestRetries } = require('./scenarios/auto-test-retries')
+const { runTestManagement } = require('./scenarios/test-management')
+const { runCiWiring } = require('./scenarios/ci-wiring')
+const { cleanupGeneratedFiles } = require('./generated-files')
+const { verifyGeneratedTestStrategy } = require('./generated-verifier')
+const { annotateCiDiscovery } = require('./ci-discovery')
+const { loadManifest } = require('./manifest-loader')
+const { createManifestScaffold } = require('./manifest-scaffold')
+const { formatExecutionPlan, getApprovalSummaryPath, getExecutionPlanPath } = require('./plan-writer')
+const { runFrameworkPreflight } = require('./preflight-runner')
+const { sanitizeConsoleText } = require('./redaction')
+const { writePendingReport, writeReport } = require('./report-writer')
+const { ensureSafeDirectory } = require('./safe-files')
+const { runSetupCommands } = require('./setup-runner')
+const {
+  getStaticBlocker,
+  runStaticDiagnosis,
+} = require('./static-diagnosis')
+
+const DEFAULT_MANIFEST = './dd-test-optimization-validation-manifest.json'
+const DEFAULT_OUT = './dd-test-optimization-validation-results'
+
+const SCENARIOS = {
+  'basic-reporting': runBasicReporting,
+  efd: runEarlyFlakeDetection,
+  atr: runAutoTestRetries,
+  'test-management': runTestManagement,
+}
+const BASIC_REPORTING_SCENARIO = 'basic-reporting'
+const CI_WIRING_SCENARIO = 'ci-wiring'
+
+function parseArgs (argv) {
+  const options = {
+    manifest: DEFAULT_MANIFEST,
+    out: DEFAULT_OUT,
+    frameworks: new Set(),
+    scenarios: new Set(getSelectableScenarios()),
+    requestedScenario: null,
+    keepTempFiles: false,
+    verbose: false,
+    approvalOverrides: [],
+  }
+
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i]
+    switch (arg) {
+      case '--manifest':
+        options.manifest = requireValue(argv, ++i, arg)
+        options.approvalOverrides.push(arg)
+        break
+      case '--out':
+        options.out = requireValue(argv, ++i, arg)
+        options.approvalOverrides.push(arg)
+        break
+      case '--framework':
+        options.frameworks.add(normalizeFrameworkTarget(requireValue(argv, ++i, arg)))
+        options.approvalOverrides.push(arg)
+        break
+      case '--scenario':
+        options.requestedScenario = requireValue(argv, ++i, arg)
+        options.scenarios = normalizeScenarioSelection(options.requestedScenario)
+        options.approvalOverrides.push(arg)
+        break
+      case '--keep-temp-files':
+        options.keepTempFiles = true
+        options.approvalOverrides.push(arg)
+        break
+      case '--verbose':
+        options.verbose = true
+        options.approvalOverrides.push(arg)
+        break
+      case '--validate-manifest':
+        options.validateManifest = true
+        break
+      case '--init-manifest':
+        options.initManifest = true
+        break
+      case '--print-plan':
+        options.printPlan = true
+        break
+      case '--print-approval-sha256':
+        options.printApprovalSha256 = true
+        break
+      case '--approved-plan-sha256':
+        options.approvedPlanSha256 = requireValue(argv, ++i, arg)
+        break
+      case '--offline-fixture-nonce':
+        options.offlineFixtureNonce = requireValue(argv, ++i, arg)
+        break
+      case '--run-approved-plan':
+        options.runApprovedPlan = requireValue(argv, ++i, arg)
+        break
+      case '--sha256':
+        options.approvedArtifactSha256 = requireValue(argv, ++i, arg)
+        break
+      case '--help':
+      case '-h':
+        options.help = true
+        break
+      default:
+        throw new Error(`Unknown argument: ${arg}`)
+    }
+  }
+
+  for (const scenario of options.scenarios) {
+    if (!getSelectableScenarios().includes(scenario)) {
+      throw new Error(`Unknown scenario "${scenario}". Expected one of: ${getSelectableScenarios().join(', ')}`)
+    }
+  }
+
+  return options
+}
+
+function requireValue (argv, index, flag) {
+  if (!argv[index]) {
+    throw new Error(`${flag} requires a value`)
+  }
+  return argv[index]
+}
+
+function printHelp () {
+  console.log(`Usage:
+  node ci/validate-test-optimization.js [options]
+
+Options:
+  --manifest <path>       Manifest path. Defaults to ${DEFAULT_MANIFEST}
+  --out <path>            Output directory. Defaults to ${DEFAULT_OUT}
+  --framework <id>        Run one framework entry. Can be repeated. A trailing ":" is ignored.
+                          A framework kind such as "vitest" runs all matching Vitest entries.
+  --scenario <name>       Run one scenario: ${getSelectableScenarios().join(', ')}
+  --keep-temp-files       Leave generated validation files in place.
+  --verbose               Print command progress.
+  --validate-manifest     Validate the manifest and exit without running project code.
+  --init-manifest         Create a schema-valid manifest scaffold without running project code.
+  --print-plan            Write the plan and approval artifacts without running project code.
+  --run-approved-plan     Run the exact approval.json produced by --print-plan.
+  --sha256 <digest>       Require approval.json and reconstructed current inputs to match this SHA-256.
+  --help                  Show this help.
+`)
+}
+
+async function main (argv) {
+  try {
+    const options = parseArgs(argv)
+    if (options.help) {
+      printHelp()
+      return
+    }
+
+    if (options.initManifest) {
+      const manifestPath = path.resolve(options.manifest)
+      if (path.dirname(manifestPath) !== process.cwd()) {
+        throw new Error('The generated manifest must be stored directly in the current repository root.')
+      }
+      const manifest = createManifestScaffold({ root: process.cwd(), frameworks: options.frameworks })
+      fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'wx' })
+      console.log(sanitizeConsoleText(
+        `Created validation manifest scaffold without running project code: ${manifestPath}\n` +
+        'This scaffold is already schema-valid; preserve its command boilerplate. Review the selected test commands ' +
+        'and CI files listed in ciDiscovery, record one replayable CI test step when available, then run ' +
+        '--validate-manifest and follow its field-specific errors.'
+      ))
+      return
+    }
+
+    if (options.runApprovedPlan) applyApprovedPlanOptions(options)
+
+    const manifest = loadManifest(options.manifest)
+    if (options.printPlan) {
+      const out = validateOutputPath(manifest, options.out)
+      const approvalManifest = getApprovalManifest(manifest, options.frameworks)
+      formatExecutionPlan({
+        manifest: approvalManifest,
+        out,
+        selectedFrameworkIds: options.frameworks.size > 0
+          ? approvalManifest.frameworks.map(framework => framework.id)
+          : [],
+        requestedScenario: options.requestedScenario,
+        keepTempFiles: options.keepTempFiles,
+        verbose: options.verbose,
+      })
+      console.log(sanitizeConsoleText(
+        '[test-optimization-validator-agent] Customer approval summary written to ' +
+        `${getApprovalSummaryPath(out)}. Read that file and send its complete contents in a user-facing message ` +
+        `before requesting approval. Detailed audit information is in ${getExecutionPlanPath(out)}. ` +
+        'The approval command is available only in the summary.'
+      ))
+      return
+    }
+    if (options.printApprovalSha256) {
+      if (!options.offlineFixtureNonce) {
+        throw new Error('--print-approval-sha256 requires the --offline-fixture-nonce value shown in the plan.')
+      }
+      const out = validateOutputPath(manifest, options.out)
+      const approvalManifest = getApprovalManifest(manifest, options.frameworks)
+      console.log(getApprovalDigest({
+        manifest: approvalManifest,
+        out,
+        selectedFrameworkIds: options.frameworks.size > 0
+          ? approvalManifest.frameworks.map(framework => framework.id)
+          : [],
+        requestedScenario: options.requestedScenario,
+        offlineFixtureNonce: options.offlineFixtureNonce,
+        keepTempFiles: options.keepTempFiles,
+        verbose: options.verbose,
+      }))
+      return
+    }
+    if (options.validateManifest) {
+      console.log(sanitizeConsoleText(`Validation manifest is valid: ${manifest.__path}`))
+      return
+    }
+    if (!options.approvedPlanSha256 || !options.offlineFixtureNonce) {
+      throw new Error(
+        'Live validation requires --run-approved-plan and --sha256 from a reviewed --print-plan result. Render and ' +
+        'approve a fresh execution plan first.'
+      )
+    }
+    const out = validateOutputPath(manifest, options.out)
+    options.repositoryRoot = manifest.repository.root
+    const selectedFrameworks = filterFrameworks(manifest.frameworks, options.frameworks)
+    const approvalManifest = getApprovalManifest(manifest, options.frameworks)
+    assertApprovalDigest(options.approvedPlanSha256, {
+      manifest: approvalManifest,
+      out,
+      selectedFrameworkIds: options.frameworks.size > 0
+        ? selectedFrameworks.map(framework => framework.id)
+        : [],
+      requestedScenario: options.requestedScenario,
+      offlineFixtureNonce: options.offlineFixtureNonce,
+      keepTempFiles: options.keepTempFiles,
+      verbose: options.verbose,
+    })
+    options.requireExecutableApproval = true
+    ensureSafeDirectory(manifest.repository.root, out, 'validation output directory', { allowRootSymlink: true })
+    if (writePendingReport) writePendingReport({ manifest, out })
+    const staticDiagnosis = runStaticDiagnosis({ manifest, out })
+    annotateCiDiscovery({ manifest, diagnosis: staticDiagnosis.report })
+
+    const results = []
+    const runnableFrameworks = []
+
+    try {
+      const frameworks = filterFrameworks(manifest.frameworks, options.frameworks)
+      const liveReadyFrameworks = []
+
+      for (const framework of frameworks) {
+        if (framework.status !== 'runnable') {
+          results.push(getFrameworkStatusResult(framework))
+          continue
+        }
+
+        const staticBlocker = getStaticBlocker(framework, staticDiagnosis.report)
+        if (staticBlocker) {
+          results.push(getStaticFailure(framework, staticBlocker, staticDiagnosis.reportPath))
+          continue
+        }
+
+        liveReadyFrameworks.push(framework)
+      }
+
+      for (const framework of liveReadyFrameworks) {
+        // Setup commands are project preparation, not Test Optimization signal collection.
+        if (framework.setup?.commands?.length > 0) logPhaseStart(framework, 'Project setup')
+        // eslint-disable-next-line no-await-in-loop
+        const setup = await runSetupCommands({ framework, out, options })
+        if (framework.setup?.commands?.length > 0) {
+          logPhaseComplete(framework, 'Project setup', setup.ok ? 'pass' : setup.failure?.status)
+        }
+        if (!setup.ok) {
+          results.push(setup.failure)
+          continue
+        }
+
+        runnableFrameworks.push(framework)
+      }
+      for (const framework of runnableFrameworks) {
+        let basicResult
+        if (options.scenarios.has(BASIC_REPORTING_SCENARIO)) {
+          // The validator owns the dd-trace-less control so ambient agent initialization cannot contaminate it.
+          logPhaseStart(framework, 'Test execution without Datadog')
+          // eslint-disable-next-line no-await-in-loop
+          const preflight = await runFrameworkPreflight({ framework, out, options })
+          logPhaseComplete(
+            framework,
+            'Test execution without Datadog',
+            preflight.ok ? 'pass' : preflight.failure?.status
+          )
+          // Scenarios intentionally run in order so each one can use an isolated offline fixture.
+          if (preflight.ok) {
+            logPhaseStart(framework, 'Basic Reporting')
+            // eslint-disable-next-line no-await-in-loop
+            basicResult = await SCENARIOS[BASIC_REPORTING_SCENARIO]({ manifest, framework, out, options })
+            logPhaseComplete(framework, 'Basic Reporting', basicResult.status)
+          } else {
+            basicResult = preflight.failure
+          }
+          results.push(basicResult)
+        }
+
+        if (options.scenarios.has(CI_WIRING_SCENARIO)) {
+          if (basicResult && basicResult.status !== 'pass') {
+            results.push(getSkippedCiWiringAfterBasicFailure(framework, basicResult))
+          } else {
+            // CI wiring runs after Basic Reporting proves this framework can report when initialized directly.
+            logPhaseStart(framework, 'CI wiring')
+            // eslint-disable-next-line no-await-in-loop
+            const ciWiringResult = await runCiWiring({ manifest, framework, out, options, basicResult })
+            results.push(ciWiringResult)
+            logPhaseComplete(framework, 'CI wiring', ciWiringResult.status)
+          }
+        }
+
+        const advancedScenarios = getAdvancedScenarios(options.scenarios)
+        if (basicResult && basicResult.status !== 'pass') {
+          for (const scenario of advancedScenarios) {
+            results.push(getSkippedAfterBasicFailure(framework, scenario, basicResult))
+          }
+          continue
+        }
+
+        if (advancedScenarios.length > 0) {
+          logPhaseStart(framework, 'Temporary test verification')
+          // eslint-disable-next-line no-await-in-loop
+          const generatedVerification = await verifyGeneratedTestStrategy({ framework, out, options })
+          logPhaseComplete(
+            framework,
+            'Temporary test verification',
+            generatedVerification.ok ? 'pass' : generatedVerification.failure?.status
+          )
+          if (!generatedVerification.ok) {
+            results.push(generatedVerification.failure)
+            for (const scenario of advancedScenarios) {
+              results.push(getSkippedAfterGeneratedVerificationFailure(
+                framework,
+                scenario,
+                generatedVerification.failure
+              ))
+            }
+            continue
+          }
+        }
+
+        for (const scenario of advancedScenarios) {
+          const runScenario = SCENARIOS[scenario]
+          // Scenarios intentionally run in order so each one can use an isolated offline fixture.
+          logPhaseStart(framework, getScenarioDisplayName(scenario))
+          // eslint-disable-next-line no-await-in-loop
+          const result = await runScenario({ manifest, framework, out, options })
+          results.push(result)
+          logPhaseComplete(framework, getScenarioDisplayName(scenario), result.status)
+        }
+      }
+    } finally {
+      await cleanupGeneratedFiles(manifest, { keep: options.keepTempFiles })
+    }
+
+    addMissingRequiredResults(results, runnableFrameworks, options.scenarios)
+    const validatorExitCode = results.some(isUnsuccessfulResult) || !didRunLiveValidation(results) ? 1 : 0
+    await writeReport({
+      manifest,
+      results,
+      out,
+      staticDiagnosis,
+      runSummary: {
+        runCompleted: true,
+        validatorExitCode,
+        validationCoverage: getValidationCoverage({
+          results,
+          requestedScenario: options.requestedScenario,
+          frameworks: selectedFrameworks,
+          scenarios: options.scenarios,
+        }),
+        checkedScenarios: [...options.scenarios],
+        omittedScenarios: getSelectableScenarios().filter(scenario => !options.scenarios.has(scenario)),
+        requestedScenario: options.requestedScenario,
+      },
+    })
+    process.exitCode = validatorExitCode
+  } catch (err) {
+    process.exitCode = 1
+    console.error(sanitizeConsoleText(err && err.stack ? err.stack : err))
+  }
+}
+
+/**
+ * Reconstructs live options from a hash-verified approval artifact.
+ *
+ * @param {object} options parsed CLI options
+ * @returns {void}
+ */
+function applyApprovedPlanOptions (options) {
+  if (!options.approvedArtifactSha256) {
+    throw new Error('--run-approved-plan requires --sha256 from the reviewed execution plan.')
+  }
+  if (options.approvalOverrides.length > 0 || options.offlineFixtureNonce || options.approvedPlanSha256) {
+    throw new Error(
+      '--run-approved-plan cannot be combined with manifest, output, selection, or legacy approval flags.'
+    )
+  }
+
+  const { material } = loadApprovedPlan(options.runApprovedPlan, options.approvedArtifactSha256)
+  options.manifest = material.manifest.path
+  options.out = material.validation.outputDirectory
+  options.frameworks = new Set(material.selection.frameworks.map(normalizeFrameworkTarget))
+  options.requestedScenario = material.selection.scenario
+  options.scenarios = options.requestedScenario
+    ? normalizeScenarioSelection(options.requestedScenario)
+    : new Set(getSelectableScenarios())
+  options.offlineFixtureNonce = material.validation.offlineFixtureNonce
+  options.keepTempFiles = material.validation.keepTemporaryFiles === true
+  options.verbose = material.validation.verbose === true
+  options.approvedPlanSha256 = options.approvedArtifactSha256
+}
+
+function validateOutputPath (manifest, outputPath) {
+  const root = path.resolve(manifest.repository.root)
+  const out = path.resolve(outputPath)
+  const relative = path.relative(root, out)
+  if (relative === '' || !relative || relative.startsWith('..') || path.isAbsolute(relative)) {
+    throw new Error('Validation output directory must be a dedicated child directory inside repository.root.')
+  }
+  return out
+}
+
+function filterFrameworks (frameworks, targets) {
+  if (targets.size === 0) return frameworks
+
+  const selected = frameworks.filter(framework => {
+    return targets.has(framework.id) || targets.has(framework.framework)
+  })
+
+  if (selected.length === 0) {
+    throw new Error(`No framework entries matched ${formatFrameworkTargets(targets)}. Available entries: ${
+      frameworks.map(framework => framework.id).join(', ') || 'none'
+    }`)
+  }
+
+  return selected
+}
+
+/**
+ * Creates the manifest view covered by a framework-scoped approval.
+ *
+ * @param {object} manifest loaded manifest
+ * @param {Set<string>} targets selected framework targets
+ * @returns {object} approval manifest
+ */
+function getApprovalManifest (manifest, targets) {
+  const frameworks = filterFrameworks(manifest.frameworks, targets)
+  if (frameworks === manifest.frameworks) return manifest
+
+  const approvalManifest = { ...manifest, frameworks }
+  Object.defineProperty(approvalManifest, '__sourceSha256', {
+    configurable: false,
+    enumerable: false,
+    value: manifest.__sourceSha256,
+    writable: false,
+  })
+  return approvalManifest
+}
+
+function normalizeFrameworkTarget (target) {
+  const normalized = String(target).trim().replaceAll(/:+$/g, '')
+  if (!normalized) {
+    throw new Error('Framework target cannot be empty')
+  }
+  return normalized
+}
+
+function formatFrameworkTargets (targets) {
+  return [...targets].map(target => `"${target}"`).join(', ')
+}
+
+function normalizeScenarioSelection (scenario) {
+  if (scenario === BASIC_REPORTING_SCENARIO) return new Set([scenario])
+  return new Set([BASIC_REPORTING_SCENARIO, scenario])
+}
+
+function getAdvancedScenarios (scenarios) {
+  return Object.keys(SCENARIOS).filter(scenario => {
+    return scenario !== BASIC_REPORTING_SCENARIO && scenarios.has(scenario)
+  })
+}
+
+/**
+ * Fails closed when orchestration omits a selected check for a runnable framework.
+ *
+ * @param {object[]} results collected validation results
+ * @param {object[]} frameworks runnable frameworks whose live phases started
+ * @param {Set<string>} scenarios selected scenarios
+ * @returns {void}
+ */
+function addMissingRequiredResults (results, frameworks, scenarios) {
+  for (const framework of frameworks) {
+    for (const scenario of scenarios) {
+      if (results.some(result => result.frameworkId === framework.id && result.scenario === scenario)) continue
+      results.push({
+        frameworkId: framework.id,
+        scenario,
+        status: 'error',
+        diagnosis: `${getScenarioDisplayName(scenario)} was selected but produced no validation result.`,
+        evidence: {
+          validationIncomplete: true,
+          recommendation: 'Rerun the validator. If the check remains absent, report this validator orchestration ' +
+            'error instead of treating the validation as successful.',
+        },
+        artifacts: [],
+      })
+    }
+  }
+}
+
+/**
+ * Reports whether all default checks produced results in an unscoped run.
+ *
+ * @param {object} input coverage inputs
+ * @param {object[]} input.results validation results
+ * @param {string|null} input.requestedScenario explicitly selected scenario
+ * @param {object[]} input.frameworks selected manifest frameworks
+ * @param {Set<string>} input.scenarios selected scenarios
+ * @returns {'complete'|'partial'} validation coverage
+ */
+function getValidationCoverage ({ results, requestedScenario, frameworks, scenarios }) {
+  if (requestedScenario) return 'partial'
+  if (results.some(result => result.evidence?.manifestIncomplete || result.evidence?.validationIncomplete)) {
+    return 'partial'
+  }
+
+  const runnableFrameworks = frameworks.filter(framework => framework.status === 'runnable')
+  if (runnableFrameworks.length === 0) return 'partial'
+  for (const framework of runnableFrameworks) {
+    for (const scenario of scenarios) {
+      if (!results.some(result => result.frameworkId === framework.id && result.scenario === scenario)) {
+        return 'partial'
+      }
+    }
+  }
+  return 'complete'
+}
+
+function getSelectableScenarios () {
+  return [
+    BASIC_REPORTING_SCENARIO,
+    CI_WIRING_SCENARIO,
+    ...Object.keys(SCENARIOS).filter(scenario => scenario !== BASIC_REPORTING_SCENARIO),
+  ]
+}
+
+function getSkippedCiWiringAfterBasicFailure (framework, basicResult) {
+  return {
+    frameworkId: framework.id,
+    scenario: 'ci-wiring',
+    status: 'skip',
+    diagnosis: 'Skipped CI wiring validation because Basic Reporting did not pass with direct Datadog ' +
+      'initialization. Fix the selected test command or local Test Optimization capability before diagnosing CI ' +
+      'wiring.',
+    evidence: {
+      blockedBy: BASIC_REPORTING_SCENARIO,
+      basicReportingStatus: basicResult.status,
+      basicReportingDiagnosis: basicResult.diagnosis,
+      featureEligibility: {
+        eligible: false,
+        blockedBy: BASIC_REPORTING_SCENARIO,
+        reasonCode: 'basic-reporting-failed',
+        scenario: 'ci-wiring',
+      },
+      ciWiring: framework.ciWiring,
+    },
+    artifacts: [],
+  }
+}
+
+function getSkippedAfterBasicFailure (framework, scenario, basicResult) {
+  return {
+    frameworkId: framework.id,
+    scenario,
+    status: 'skip',
+    diagnosis: `Skipped because basic reporting did not pass: ${basicResult.diagnosis}`,
+    evidence: {
+      blockedBy: BASIC_REPORTING_SCENARIO,
+      basicReportingStatus: basicResult.status,
+      basicReportingDiagnosis: basicResult.diagnosis,
+      featureEligibility: {
+        eligible: false,
+        blockedBy: BASIC_REPORTING_SCENARIO,
+        reasonCode: 'basic-reporting-failed',
+        scenario,
+      },
+    },
+    artifacts: [],
+  }
+}
+
+function getSkippedAfterGeneratedVerificationFailure (framework, scenario, failure) {
+  return {
+    frameworkId: framework.id,
+    scenario,
+    status: 'skip',
+    diagnosis: `Skipped because the temporary validation test could not run as expected: ${failure.diagnosis}`,
+    evidence: {
+      blockedBy: 'generated-test-verification',
+      verificationStatus: failure.status,
+      verificationDiagnosis: failure.diagnosis,
+      featureEligibility: {
+        eligible: false,
+        blockedBy: 'generated-test-verification',
+        reasonCode: 'generated-test-verification-failed',
+        scenario,
+      },
+    },
+    artifacts: [],
+  }
+}
+
+function isUnsuccessfulResult (result) {
+  return result.status === 'fail' || result.status === 'error' || result.status === 'blocked'
+}
+
+/**
+ * Reports whether at least one live validation check produced a result.
+ *
+ * Framework discovery-only entries use the synthetic `all` scenario and do not prove Test Optimization behavior.
+ *
+ * @param {object[]} results validation results
+ * @returns {boolean} whether live validation ran
+ */
+function didRunLiveValidation (results) {
+  return results.some(result => result.scenario !== 'all')
+}
+
+/**
+ * Prints the start of one framework validation phase.
+ *
+ * @param {object} framework manifest framework entry
+ * @param {string} phase customer-facing phase name
+ * @returns {void}
+ */
+function logPhaseStart (framework, phase) {
+  logValidationProgress(`${framework.id}: ${phase} started.`)
+}
+
+/**
+ * Prints the outcome of one framework validation phase.
+ *
+ * @param {object} framework manifest framework entry
+ * @param {string} phase customer-facing phase name
+ * @param {string|undefined} status phase outcome
+ * @returns {void}
+ */
+function logPhaseComplete (framework, phase, status) {
+  logValidationProgress(`${framework.id}: ${phase} ${status || 'complete'}.`)
+}
+
+/**
+ * Prints a sanitized validator progress line.
+ *
+ * @param {string} message progress message
+ * @returns {void}
+ */
+function logValidationProgress (message) {
+  console.log(sanitizeConsoleText(`[test-optimization-validator] ${message}`))
+}
+
+/**
+ * Converts an advanced scenario id to customer-facing text.
+ *
+ * @param {string} scenario scenario id
+ * @returns {string} display name
+ */
+function getScenarioDisplayName (scenario) {
+  return {
+    'basic-reporting': 'Basic Reporting',
+    'ci-wiring': 'CI Wiring',
+    efd: 'Early Flake Detection',
+    atr: 'Auto Test Retries',
+    'test-management': 'Test Management',
+  }[scenario] || scenario
+}
+
+function getStaticFailure (framework, blocker, staticDiagnosisPath) {
+  return {
+    frameworkId: framework.id,
+    scenario: 'all',
+    status: 'fail',
+    diagnosis: blocker.reason,
+    evidence: {
+      staticDiagnosis: true,
+      recommendation: blocker.recommendation,
+    },
+    artifacts: [staticDiagnosisPath],
+  }
+}
+
+function getFrameworkStatusResult (framework) {
+  const evidence = getFrameworkStatusEvidence(framework)
+
+  if (framework.status === 'unsupported_by_validator') {
+    const frameworkName = getDisplayFrameworkName(framework.framework)
+
+    return {
+      frameworkId: framework.id,
+      scenario: 'all',
+      status: 'skip',
+      diagnosis: `${frameworkName} is not supported as a Test Optimization test framework.`,
+      evidence: {
+        ...evidence,
+        recommendation: 'Choose a supported framework before running live validation.',
+      },
+      artifacts: [],
+    }
+  }
+
+  return {
+    frameworkId: framework.id,
+    scenario: 'all',
+    status: 'skip',
+    diagnosis: getFrameworkStatusDiagnosis(framework, evidence),
+    evidence: {
+      ...evidence,
+      recommendation: 'Provide a small runnable command for this framework, or mark the setup blocker explicitly.',
+    },
+    artifacts: [],
+  }
+}
+
+function getFrameworkStatusDiagnosis (framework, evidence) {
+  const frameworkName = framework.framework
+  const notes = evidence.manifestNotes || []
+
+  if (isDependencyOnlyDetection(evidence)) {
+    return getDependencyOnlyDiagnosis(framework, evidence)
+  }
+
+  if (notes.length > 0) {
+    return `${frameworkName} was detected, but no runnable validation command was available. ` +
+      `Basic reporting was not run. Manifest reason: ${notes[0]}`
+  }
+
+  return `${frameworkName} was detected, but the manifest did not prove a runnable validation command. ` +
+    'Basic reporting was not run. See discovery evidence for scripts/config files to turn into a small command.'
+}
+
+function isDependencyOnlyDetection (evidence) {
+  return evidence.directDependency && evidence.configFiles.length === 0 && evidence.frameworkScripts.length === 0
+}
+
+function getDependencyOnlyDiagnosis (framework, evidence) {
+  const frameworkName = getDisplayFrameworkName(framework.framework)
+  const dependency = formatDependency(evidence.directDependency)
+  const note = evidence.manifestNotes?.[0] ? ` Manifest note: ${evidence.manifestNotes[0]}` : ''
+  const common = `${frameworkName} is installed${dependency}, but this repository does not appear to use ` +
+    `${frameworkName} to run tests: no ${framework.framework} config, package script, or runnable ` +
+    `${framework.framework} test command was found. Basic reporting was not run for ${frameworkName}.`
+
+  if (framework.framework === 'playwright') {
+    return `${frameworkName} is installed${dependency}, but no Playwright Test setup was found. ` +
+      'The playwright package can be used only for browser automation; Test Optimization validation needs a ' +
+      '`playwright test` setup with a config, script, or runnable test command. Basic reporting was not run ' +
+      `for ${frameworkName}.${note}`
+  }
+
+  return `${common} If this repo does use ${frameworkName}, provide a small ${frameworkName} test command; ` +
+    `otherwise this dependency-only detection can be ignored.${note}`
+}
+
+function getDisplayFrameworkName (frameworkName) {
+  return {
+    cucumber: 'Cucumber',
+    cypress: 'Cypress',
+    jest: 'Jest',
+    mocha: 'Mocha',
+    playwright: 'Playwright',
+    vitest: 'Vitest',
+  }[frameworkName] || frameworkName
+}
+
+function formatDependency (dependency) {
+  if (!dependency) return ''
+  return ` in ${dependency.field}${dependency.version ? ` (${dependency.version})` : ''}`
+}
+
+function getFrameworkStatusEvidence (framework) {
+  const root = framework.project?.root
+  return {
+    frameworkStatus: framework.status,
+    frameworkVersion: framework.frameworkVersion,
+    manifestNotes: Array.isArray(framework.notes) ? framework.notes : [],
+    directDependency: root ? getDirectDependency(root, framework.framework) : undefined,
+    frameworkScripts: root ? findFrameworkScripts(root, framework.framework) : [],
+    testLikeScripts: root ? findTestLikeScripts(root) : [],
+    configFiles: root ? findFrameworkConfigFiles(root, framework.framework) : [],
+  }
+}
+
+function getDirectDependency (root, frameworkName) {
+  const packageJson = readPackageJson(root)
+  if (!packageJson) return
+
+  for (const field of ['dependencies', 'devDependencies', 'optionalDependencies']) {
+    const value = packageJson[field]?.[frameworkName]
+    if (value) return { field, version: value }
+  }
+}
+
+function findFrameworkScripts (root, frameworkName) {
+  return findScripts(root, (name, command) => {
+    return includesWord(name, frameworkName) || includesWord(command, frameworkName)
+  })
+}
+
+function findTestLikeScripts (root) {
+  return findScripts(root, name => /(^|:)(test|unit|e2e|integration|ci)(:|$)/.test(name)).slice(0, 8)
+}
+
+function findScripts (root, predicate) {
+  const packageJson = readPackageJson(root)
+  const scripts = packageJson?.scripts || {}
+  const matches = []
+  for (const [name, command] of Object.entries(scripts)) {
+    if (predicate(name, command)) matches.push({ name, command })
+  }
+  return matches.slice(0, 8)
+}
+
+function findFrameworkConfigFiles (root, frameworkName) {
+  const patterns = getFrameworkConfigPatterns(frameworkName)
+  if (patterns.length === 0) return []
+
+  const files = []
+  findFiles(root, 4, file => {
+    if (patterns.some(pattern => pattern.test(path.basename(file)))) {
+      files.push(path.relative(root, file))
+    }
+    return files.length < 8
+  })
+  return files
+}
+
+function getFrameworkConfigPatterns (frameworkName) {
+  const definition = getFrameworkDefinitions(DD_MAJOR).find(definition => definition.id === frameworkName)
+  return definition?.configPatterns || []
+}
+
+function readPackageJson (root) {
+  try {
+    return JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'))
+  } catch {
+    return null
+  }
+}
+
+function findFiles (dir, depth, visit) {
+  if (depth < 0) return true
+
+  let entries
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true })
+  } catch {
+    return true
+  }
+
+  for (const entry of entries) {
+    if (shouldSkipDirectory(entry.name)) continue
+    const filename = path.join(dir, entry.name)
+    if (entry.isDirectory()) {
+      if (!findFiles(filename, depth - 1, visit)) return false
+    } else if (!visit(filename)) {
+      return false
+    }
+  }
+
+  return true
+}
+
+function shouldSkipDirectory (name) {
+  return name === '.git' || name === 'node_modules' || name === 'dist' || name === 'coverage'
+}
+
+function includesWord (value, word) {
+  return new RegExp(`(^|[^a-zA-Z0-9_-])${escapeRegExp(word)}([^a-zA-Z0-9_-]|$)`).test(value)
+}
+
+function escapeRegExp (value) {
+  return value.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`)
+}
+
+module.exports = { filterFrameworks, main, normalizeFrameworkTarget, parseArgs }