dd-trace 6.4.0 → 6.5.0

package/ci/test-optimization-validation/scenarios/test-management.js

@@ -0,0 +1,218 @@
+'use strict'
+
+const path = require('path')
+
+const {
+  discoverScenarioTests,
+  discoveryEvidence,
+  error,
+  failWithDebugRerun,
+  pass,
+  prepareGeneratedScenario,
+  requireGeneratedScenario,
+  runInstrumentedCommand,
+  testEventSamples,
+  testsForDiscoveredScenario,
+} = require('./helpers')
+
+async function runTestManagement ({ framework, out, options }) {
+  const scenarioName = 'test-management'
+  const skipResult = requireGeneratedScenario(framework, 'test-management-target', scenarioName)
+  if (skipResult) return skipResult
+
+  let outDir
+  try {
+    const { scenario } = await prepareGeneratedScenario(framework, 'test-management-target')
+    const discovery = await discoverScenarioTests({ framework, out, scenarioName, scenario, options })
+    if (discovery.tests.length === 0) {
+      return failWithDebugRerun({
+        command: scenario.runCommand,
+        diagnosis: 'The test-management target was not reported during baseline identity discovery.',
+        evidence: discoveryEvidence(discovery),
+        framework,
+        options,
+        out,
+        outDir: discovery.outDir,
+        scenarioName,
+      })
+    }
+
+    const testManagementTests = buildQuarantinedResponse(framework, scenario, discovery.testIdentities)
+    const fixtureConfig = {
+      settings: {
+        test_management: {
+          enabled: true,
+          attempt_to_fix_retries: 2,
+        },
+      },
+      testManagementTests,
+    }
+
+    const run = await runInstrumentedCommand({
+      framework,
+      out,
+      scenarioName,
+      command: scenario.runCommand,
+      options,
+      fixtureConfig,
+    })
+    outDir = run.outDir
+
+    const tests = testsForDiscoveredScenario(run.events, scenario, discovery)
+    const quarantinedTests = tests.filter(test => test.isQuarantined)
+    const evidence = {
+      ...discoveryEvidence(discovery),
+      commandExitCode: run.result.exitCode,
+      commandTimedOut: run.result.timedOut,
+      settingsLoadedFromCache: run.offline.inputs.settings?.status === 'loaded',
+      testManagementLoadedFromCache: run.offline.inputs.test_management?.status === 'loaded',
+      configuredManagedTests: summarizeManagedTests(testManagementTests),
+      matchingTestEvents: tests.length,
+      quarantinedEvents: quarantinedTests.length,
+      samples: testEventSamples(tests),
+    }
+
+    if (run.result.exitCode !== 0) {
+      return failWithDebugRerun({
+        command: scenario.runCommand,
+        fixtureConfig,
+        diagnosis: 'The generated Test Management command reported quarantined-test evidence, but the command ' +
+          `exited ${run.result.exitCode}. Test Management is only valid when the command completes successfully ` +
+          'with the managed test applied.',
+        evidence,
+        framework,
+        options,
+        out,
+        outDir,
+        scenarioName,
+      })
+    }
+
+    if (!evidence.settingsLoadedFromCache || !evidence.testManagementLoadedFromCache) {
+      return failWithDebugRerun({
+        command: scenario.runCommand,
+        fixtureConfig,
+        diagnosis: 'Test Management settings or managed-test data were not loaded from the offline cache fixture.',
+        evidence,
+        framework,
+        options,
+        out,
+        outDir,
+        scenarioName,
+      })
+    }
+
+    if (tests.length === 0) {
+      return failWithDebugRerun({
+        command: scenario.runCommand,
+        fixtureConfig,
+        diagnosis: 'The test-management target test was not reported.',
+        evidence,
+        framework,
+        options,
+        out,
+        outDir,
+        scenarioName,
+      })
+    }
+
+    if (quarantinedTests.length === 0) {
+      return failWithDebugRerun({
+        command: scenario.runCommand,
+        fixtureConfig,
+        diagnosis: 'Test Management was enabled, but the generated target was not tagged as quarantined.',
+        evidence,
+        framework,
+        options,
+        out,
+        outDir,
+        scenarioName,
+      })
+    }
+
+    return pass(
+      framework,
+      scenarioName,
+      'The generated target test was matched by Test Management and tagged as quarantined.',
+      evidence,
+      outDir
+    )
+  } catch (err) {
+    return error(framework, scenarioName, err, outDir)
+  }
+}
+
+function buildQuarantinedResponse (framework, scenario, discoveredIdentities = []) {
+  const suites = {}
+  const identities = [...(scenario.testIdentities || []), ...discoveredIdentities]
+  for (const identity of identities) {
+    for (const suite of getSuiteCandidates(identity, scenario)) {
+      for (const name of getNameCandidates(identity)) {
+        suites[suite] = suites[suite] || { tests: {} }
+        suites[suite].tests[name] = {
+          properties: {
+            quarantined: true,
+          },
+        }
+      }
+    }
+  }
+
+  return {
+    [framework.framework]: {
+      suites,
+    },
+  }
+}
+
+function getSuiteCandidates (identity, scenario) {
+  const candidates = new Set()
+  addCandidate(candidates, identity.suite)
+  addCandidate(candidates, identity.file)
+
+  if (identity.file) {
+    addCandidate(candidates, normalizePath(path.basename(identity.file)))
+    if (scenario.runCommand?.cwd) {
+      addCandidate(candidates, normalizePath(path.relative(scenario.runCommand.cwd, identity.file)))
+    }
+  }
+
+  return [...candidates]
+}
+
+function getNameCandidates (identity) {
+  const candidates = new Set()
+  addCandidate(candidates, identity.name)
+  if (!identity.discovered && identity.suite && identity.name && !identity.name.startsWith(`${identity.suite} `)) {
+    addCandidate(candidates, `${identity.suite} ${identity.name}`)
+  }
+  return [...candidates]
+}
+
+function addCandidate (candidates, value) {
+  if (value) candidates.add(normalizePath(value))
+}
+
+function normalizePath (value) {
+  return value.replaceAll(path.sep, '/')
+}
+
+function summarizeManagedTests (testManagementTests) {
+  const managed = testManagementTests && testManagementTests[Object.keys(testManagementTests)[0]]
+  const suites = managed?.suites || {}
+  const summary = new Map()
+  for (const [suite, { tests = {} }] of Object.entries(suites)) {
+    const displaySuite = path.isAbsolute(suite) ? path.basename(suite) : suite
+    const testNames = summary.get(displaySuite) || new Set()
+    for (const testName of Object.keys(tests)) {
+      testNames.add(testName)
+    }
+    summary.set(displaySuite, testNames)
+  }
+  return [...summary.entries()].slice(0, 5).map(([suite, tests]) => ({
+    suite,
+    tests: [...tests].slice(0, 5),
+  }))
+}
+
+module.exports = { runTestManagement, buildQuarantinedResponse }

package/ci/test-optimization-validation/setup-runner.js

@@ -0,0 +1,97 @@
+'use strict'
+
+const path = require('path')
+
+const { getArtifactId } = require('./artifact-id')
+const { runCommand } = require('./command-runner')
+
+async function runSetupCommands ({ framework, out, options }) {
+  const commands = framework.setup?.commands || []
+  const results = []
+  const artifacts = []
+
+  for (let index = 0; index < commands.length; index++) {
+    const command = commands[index]
+    const outDir = path.join(
+      out,
+      'setup',
+      getArtifactId(framework.id),
+      `${index + 1}-${getArtifactId(command.id || 'setup')}`
+    )
+    // eslint-disable-next-line no-await-in-loop
+    const result = await runCommand(command, {
+      artifactRoot: out,
+      envMode: 'clean',
+      outDir,
+      label: `${framework.id}:setup:${command.id || index + 1}`,
+      repositoryRoot: options.repositoryRoot,
+      requireExecutableApproval: options.requireExecutableApproval,
+      verbose: options.verbose,
+    })
+    const summary = summarizeSetupCommand(command, result, outDir)
+    results.push(summary)
+    artifacts.push(...Object.values(result.artifacts))
+
+    if (command.required !== false && result.exitCode !== 0) {
+      const failure = getSetupFailure(framework, command, result, results)
+      failure.artifacts.push(...artifacts)
+      return {
+        ok: false,
+        results,
+        artifacts,
+        failure,
+      }
+    }
+  }
+
+  return { ok: true, results, artifacts }
+}
+
+function getSetupFailure (framework, command, result, setupCommands) {
+  const setupName = command.description || command.id || result.command
+
+  return {
+    frameworkId: framework.id,
+    scenario: 'all',
+    status: 'blocked',
+    diagnosis: `Validation is blocked by required project setup: ${setupName}. ` +
+      'No Test Optimization conclusion was reached for this framework.',
+    evidence: {
+      blockedByProjectSetup: true,
+      setupFailed: true,
+      setupCommand: {
+        id: command.id,
+        description: command.description,
+        command: result.command,
+        cwd: result.cwd,
+        exitCode: result.exitCode,
+        timedOut: result.timedOut,
+        stdoutSummary: tail(result.stdout),
+        stderrSummary: tail(result.stderr),
+      },
+      setupCommands,
+      recommendation: 'Run or fix the documented project setup command, then rerun validation for this framework.',
+    },
+    artifacts: [],
+  }
+}
+
+function summarizeSetupCommand (command, result, outDir) {
+  return {
+    id: command.id,
+    description: command.description,
+    required: command.required !== false,
+    command: result.command,
+    cwd: result.cwd,
+    exitCode: result.exitCode,
+    timedOut: result.timedOut,
+    durationMs: result.durationMs,
+    artifactDirectory: outDir,
+  }
+}
+
+function tail (value) {
+  return String(value || '').trim().split(/\r?\n/).slice(-20).join('\n')
+}
+
+module.exports = { runSetupCommands }

package/ci/test-optimization-validation/static-diagnosis.js

@@ -0,0 +1,159 @@
+'use strict'
+
+const path = require('path')
+
+const satisfies = require('../../vendor/dist/semifies')
+const { DD_MAJOR } = require('../../version')
+const {
+  getFrameworkDefinitions,
+  runDiagnosis,
+} = require('../diagnose')
+const { sanitizeForReport } = require('./redaction')
+const { writeFileSafely } = require('./safe-files')
+
+const SUPPORTED_FRAMEWORKS = new Set([
+  'jest',
+  'mocha',
+  'cucumber',
+  'cypress',
+  'playwright',
+  'vitest',
+])
+
+const UNSUPPORTED_FRAMEWORK_NAMES = {
+  ava: 'AVA',
+  jasmine: 'Jasmine',
+  karma: 'Karma',
+  'node:test': 'Node.js test runner',
+  tap: 'tap',
+  testcafe: 'TestCafe',
+  uvu: 'uvu',
+}
+
+function runStaticDiagnosis ({ manifest, out }) {
+  const report = runDiagnosis({ root: manifest.repository.root, excludePaths: [out] })
+  const reportPath = path.join(out, 'static-diagnosis.json')
+  writeFileSafely(
+    out,
+    reportPath,
+    `${JSON.stringify(sanitizeForReport(report), null, 2)}\n`,
+    'static diagnosis artifact'
+  )
+  return { report, reportPath }
+}
+
+function getStaticBlocker (framework, diagnosis) {
+  if (!SUPPORTED_FRAMEWORKS.has(framework.framework)) {
+    return {
+      reason: `Unsupported test framework detected: ${getUnsupportedFrameworkName(framework)}.`,
+      recommendation: 'Choose Jest, Mocha, Cucumber, Cypress, Playwright, or Vitest for live validation.',
+    }
+  }
+
+  const definition = getFrameworkDefinition(framework, diagnosis)
+  if (!definition) return null
+
+  const version = parseVersion(framework.frameworkVersion)
+  if (version && !satisfies(version, definition.supportedRange)) {
+    return {
+      reason:
+        `${definition.name} ${version} is not supported. Supported range is ${definition.supportedRange}.`,
+      recommendation: definition.recommendation,
+    }
+  }
+
+  const staticVersionError = findStaticVersionError(definition, diagnosis, framework, version)
+  if (staticVersionError) {
+    return {
+      reason: staticVersionError.title,
+      recommendation: staticVersionError.recommendation || staticVersionError.message,
+    }
+  }
+
+  return null
+}
+
+function getFrameworkDefinition (framework, diagnosis) {
+  const definitions = getFrameworkDefinitions(diagnosis.ddTraceMajor || DD_MAJOR)
+  return definitions.find(definition => definition.id === framework.framework)
+}
+
+function getUnsupportedFrameworkName (framework) {
+  return UNSUPPORTED_FRAMEWORK_NAMES[framework.framework] || framework.framework || framework.id
+}
+
+function findStaticVersionError (definition, diagnosis, framework, manifestVersion) {
+  const results = Array.isArray(diagnosis.results) ? diagnosis.results : []
+  return results.find(result => {
+    return result.status === 'error' &&
+      typeof result.title === 'string' &&
+      result.title.startsWith(`${definition.name} `) &&
+      result.title.includes(' is not supported') &&
+      staticVersionErrorAppliesToFramework(result, diagnosis, framework, definition, manifestVersion)
+  })
+}
+
+function staticVersionErrorAppliesToFramework (result, diagnosis, framework, definition, manifestVersion) {
+  const locations = Array.isArray(result.locations) ? result.locations : []
+  if (locations.length === 0) {
+    return !(manifestVersion && satisfies(manifestVersion, definition.supportedRange))
+  }
+
+  return locations.some(location => locationMatchesFramework(location, diagnosis, framework))
+}
+
+function locationMatchesFramework (location, diagnosis, framework) {
+  const relativeLocation = normalizeRelativePath(location)
+  const exactLocations = getExactFrameworkLocations(diagnosis, framework)
+
+  if (exactLocations.has(relativeLocation)) return true
+
+  const projectRoot = getRelativeFrameworkProjectRoot(diagnosis, framework)
+  return projectRoot !== '' &&
+    (relativeLocation === projectRoot || relativeLocation.startsWith(`${projectRoot}/`))
+}
+
+function getExactFrameworkLocations (diagnosis, framework) {
+  const locations = new Set()
+  addRelativeFrameworkLocation(locations, diagnosis, framework.project?.packageJson)
+
+  for (const configFile of framework.project?.configFiles || []) {
+    addRelativeFrameworkLocation(locations, diagnosis, configFile)
+  }
+
+  return locations
+}
+
+function addRelativeFrameworkLocation (locations, diagnosis, location) {
+  if (typeof location !== 'string' || location.length === 0) return
+  locations.add(getRelativeLocation(diagnosis, location))
+}
+
+function getRelativeFrameworkProjectRoot (diagnosis, framework) {
+  return getRelativeLocation(diagnosis, framework.project?.root)
+}
+
+function getRelativeLocation (diagnosis, location) {
+  if (typeof location !== 'string' || location.length === 0) return ''
+  const root = typeof diagnosis.root === 'string' ? diagnosis.root : ''
+  const relativeLocation = path.isAbsolute(location) && root
+    ? path.relative(root, location)
+    : location
+
+  return normalizeRelativePath(relativeLocation)
+}
+
+function normalizeRelativePath (location) {
+  return location.split(path.sep).join('/').replace(/^\.\//, '')
+}
+
+function parseVersion (rawVersion) {
+  if (typeof rawVersion !== 'string') return null
+  const match = rawVersion.match(/\d+\.\d+\.\d+/)
+  return match ? match[0] : null
+}
+
+module.exports = {
+  getStaticBlocker,
+  runStaticDiagnosis,
+}

package/ci/test-optimization-validation/test-output.js

@@ -0,0 +1,129 @@
+'use strict'
+
+const ANSI_PATTERN = new RegExp(`${String.fromCharCode(27)}${String.raw`\[[0-?]*[ -/]*[@-~]`}`, 'g')
+
+/**
+ * Extracts the final test count from common JavaScript test-runner summaries.
+ *
+ * @param {string} framework test framework name
+ * @param {string} stdout captured stdout
+ * @param {string} stderr captured stderr
+ * @returns {number|null} observed count when a supported summary was found
+ */
+function getObservedTestCount (framework, stdout = '', stderr = '') {
+  const output = `${stdout}\n${stderr}`.replaceAll(ANSI_PATTERN, '')
+  if (framework === 'jest') return getJestObservedTestCount(output)
+  if (framework === 'vitest') return getVitestObservedTestCount(output)
+  if (framework === 'playwright') return getPlaywrightObservedTestCount(output)
+
+  const totalPatterns = framework === 'node:test'
+    ? [/^# tests\s+(\d+)\s*$/gim]
+    : framework === 'cucumber'
+      ? [/\b(\d+)\s+scenarios?\b/gi]
+      : []
+
+  for (const pattern of totalPatterns) {
+    const count = getLastMatchCount(output, pattern)
+    if (count !== null) return count
+  }
+
+  if (framework === 'mocha') {
+    return sumLastMatchCounts(output, [
+      /\b(\d+)\s+passing\b/gi,
+      /\b(\d+)\s+failing\b/gi,
+      /\b(\d+)\s+pending\b/gi,
+    ])
+  }
+
+  return getLastMatchCount(output, /\b(\d+)\s+tests?\s+(?:passed|failed)\b/gi)
+}
+
+/**
+ * Counts Playwright tests that completed instead of treating skipped tests as executions.
+ *
+ * @param {string} output test output without ANSI codes
+ * @returns {number|null} executed test count
+ */
+function getPlaywrightObservedTestCount (output) {
+  const observed = sumLastMatchCounts(output, [
+    /^\s*(\d+)\s+passed\b/gim,
+    /^\s*(\d+)\s+failed\b/gim,
+    /^\s*(\d+)\s+flaky\b/gim,
+  ])
+  if (observed !== null) return observed
+  return /^\s*\d+\s+skipped\b/im.test(output) ? 0 : null
+}
+
+/**
+ * Counts Jest tests that actually ran, excluding tests skipped by a name filter.
+ *
+ * @param {string} output test output without ANSI codes
+ * @returns {number|null} executed test count
+ */
+function getJestObservedTestCount (output) {
+  return getExecutedTestSummaryCount(output, /^\s*Tests:\s+/)
+}
+
+/**
+ * Counts Vitest tests that actually ran, excluding tests skipped by a name filter.
+ *
+ * @param {string} output test output without ANSI codes
+ * @returns {number|null} executed test count
+ */
+function getVitestObservedTestCount (output) {
+  return getExecutedTestSummaryCount(output, /^\s*Tests\s+/)
+}
+
+/**
+ * Extracts passed and failed counts from the final matching runner summary.
+ *
+ * @param {string} output test output without ANSI codes
+ * @param {RegExp} summaryPattern test-summary line pattern
+ * @returns {number|null} executed test count
+ */
+function getExecutedTestSummaryCount (output, summaryPattern) {
+  const summaryLines = output.split(/\r?\n/).filter(line => summaryPattern.test(line))
+  const summary = summaryLines.at(-1)
+  if (!summary) return null
+
+  const observed = sumLastMatchCounts(summary, [
+    /\b(\d+)\s+passed\b/gi,
+    /\b(\d+)\s+failed\b/gi,
+  ])
+  if (observed !== null) return observed
+  return /\b\d+\s+skipped\b/i.test(summary) ? 0 : null
+}
+
+/**
+ * Returns the count captured by the last match of one summary pattern.
+ *
+ * @param {string} output test output
+ * @param {RegExp} pattern global summary pattern
+ * @returns {number|null} final captured count
+ */
+function getLastMatchCount (output, pattern) {
+  let count = null
+  for (const match of output.matchAll(pattern)) count = Number(match[1])
+  return count
+}
+
+/**
+ * Sums the final counts from multiple summary categories.
+ *
+ * @param {string} output test output
+ * @param {RegExp[]} patterns global summary patterns
+ * @returns {number|null} summed count when any category was found
+ */
+function sumLastMatchCounts (output, patterns) {
+  let found = false
+  let count = 0
+  for (const pattern of patterns) {
+    const value = getLastMatchCount(output, pattern)
+    if (value === null) continue
+    found = true
+    count += value
+  }
+  return found ? count : null
+}
+
+module.exports = { getObservedTestCount }

package/ci/test-optimization-validation-manifest.schema.json

@@ -0,0 +1 @@
+{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://github.com/DataDog/dd-trace-js/ci/test-optimization-validation-manifest.schema.json","title":"Datadog Test Optimization validation manifest","type":"object","required":["schemaVersion","repository","environment","frameworks"],"properties":{"schemaVersion":{"type":"string"},"generatedAt":{"type":"string"},"repository":{"$ref":"#/$defs/repository"},"environment":{"$ref":"#/$defs/environment"},"ciDiscovery":{"$ref":"#/$defs/ciDiscovery"},"frameworks":{"type":"array","minItems":1,"items":{"$ref":"#/$defs/framework"}},"omitted":{"type":"array","items":{"type":"string"}},"warnings":{"type":"array","items":{"type":"string"}}},"additionalProperties":true,"$defs":{"repository":{"type":"object","required":["root"],"properties":{"root":{"$ref":"#/$defs/absolutePathString"},"gitRemote":{"type":["string","null"]},"gitSha":{"type":["string","null"]},"packageManager":{"enum":["npm","yarn","pnpm","bun","mixed","unknown"]},"workspaceManager":{"enum":["npm","yarn","pnpm","lerna","nx","turbo","rush","none","unknown"]}},"additionalProperties":true},"environment":{"type":"object","properties":{"os":{"enum":["darwin","linux","windows","unknown"]},"shell":{"type":["string","null"],"minLength":1,"pattern":"^[^\\u0000-\\u001F\\u007F-\\u009F\\u00AD\\u0600-\\u0605\\u061C\\u06DD\\u070F\\u0890\\u0891\\u08E2\\u180B-\\u180F\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u206F\\uFE00-\\uFE0F\\uFEFF\\uFFF9-\\uFFFB]*$"},"nodeVersion":{"type":["string","null"]},"requiredEnvVars":{"type":"array","items":{"type":"string"}},"safeEnv":{"type":"object"}},"additionalProperties":true},"ciDiscovery":{"type":"object","properties":{"searched":{"type":"array","items":{"type":"string"}},"found":{"type":"array","items":{"type":"string"}},"staticFound":{"type":"array","items":{"type":"string"}},"method":{"type":"string"},"warnings":{"type":"array","items":{"type":"string"}},"notes":{"type":"array","items":{"type":"string"}},"contradictions":{"type":"array","items":{"type":"string"}}},"additionalProperties":true},"framework":{"type":"object","required":["id","framework","status","project"],"properties":{"id":{"type":"string"},"framework":{"enum":["jest","vitest","mocha","cucumber","cypress","playwright","node:test","ava","tap","jasmine","karma","uvu","testcafe","custom","unknown"]},"frameworkVersion":{"type":["string","null"]},"language":{"enum":["javascript","typescript","mixed","unknown"]},"status":{"enum":["runnable","detected_not_runnable","requires_external_service","requires_manual_setup","unsupported_by_validator","unknown"]},"supportLevel":{"enum":["validator_supported","dd_trace_supported_but_validator_missing_adapter","detected_only","unknown"]},"project":{"$ref":"#/$defs/project"},"setup":{"$ref":"#/$defs/setup"},"existingTestCommand":{"$ref":"#/$defs/command"},"ciWiring":{"$ref":"#/$defs/ciWiring"},"ciWiringCommand":{"$ref":"#/$defs/command"},"forcedLocalCommand":false,"preflight":{"$ref":"#/$defs/preflight"},"generatedTestStrategy":{"$ref":"#/$defs/generatedTestStrategy"},"notes":{"type":"array","items":{"type":"string"}}},"additionalProperties":true,"allOf":[{"if":{"properties":{"status":{"const":"runnable"}}},"then":{"required":["existingTestCommand","preflight","ciWiring"]}},{"if":{"properties":{"status":{"enum":["detected_not_runnable","requires_external_service","requires_manual_setup","unsupported_by_validator","unknown"]}}},"then":{"required":["notes"],"properties":{"notes":{"minItems":1}}}},{"if":{"required":["ciWiringCommand"]},"then":{"properties":{"ciWiring":{"required":["provider","configFile","job","step","workingDirectory","whySelected"]}}}},{"if":{"required":["ciWiring"],"properties":{"ciWiring":{"required":["replayability"],"properties":{"replayability":{"const":"replayable"}}}}},"then":{"required":["ciWiringCommand"]}},{"if":{"required":["ciWiring"],"properties":{"ciWiring":{"required":["replayability"],"properties":{"replayability":{"const":"not_replayable"}}}}},"then":{"not":{"required":["ciWiringCommand"]}}}]},"project":{"type":"object","required":["root"],"properties":{"name":{"type":["string","null"]},"root":{"$ref":"#/$defs/absolutePathString"},"packageJson":{"anyOf":[{"$ref":"#/$defs/absolutePathString"},{"type":"null"}]},"configFiles":{"type":"array","items":{"$ref":"#/$defs/absolutePathString"}},"evidence":{"type":"array","items":{"type":"string"}}},"additionalProperties":true},"setup":{"type":"object","properties":{"commands":{"type":"array","items":{"$ref":"#/$defs/setupCommand"}},"services":{"type":"array","items":{"$ref":"#/$defs/service"}}},"additionalProperties":true},"setupCommand":{"allOf":[{"$ref":"#/$defs/command"},{"type":"object","properties":{"id":{"type":"string"},"description":{"type":"string"},"required":{"type":"boolean"}}}]},"service":{"type":"object","properties":{"name":{"type":"string"},"required":{"type":"boolean"},"description":{"type":"string"}},"additionalProperties":true},"command":{"type":"object","required":["cwd"],"properties":{"description":{"type":"string"},"id":{"type":"string"},"required":{"type":"boolean"},"cwd":{"$ref":"#/$defs/absolutePathString"},"argv":{"type":"array","items":{"$ref":"#/$defs/executableString"}},"env":{"type":"object","propertyNames":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$"},"additionalProperties":{"$ref":"#/$defs/executableString"}},"requiredEnvVars":{"type":"array","items":{"type":"string"}},"outputPaths":{"type":"array","items":{"$ref":"#/$defs/absolutePathString"}},"timeoutMs":{"type":"number","exclusiveMinimum":0,"maximum":1800000},"usesShell":{"type":"boolean"},"shell":{"$ref":"#/$defs/executableString"},"shellCommand":{"anyOf":[{"$ref":"#/$defs/executableString"},{"type":"null"}]},"shellReason":{"type":["string","null"]}},"additionalProperties":false,"allOf":[{"if":{"required":["usesShell"],"properties":{"usesShell":{"const":true}}},"then":{"required":["shellCommand"]}},{"if":{"required":["shell"]},"then":{"required":["usesShell"],"properties":{"usesShell":{"const":true}}}}],"anyOf":[{"required":["argv"]},{"properties":{"usesShell":{"const":true}},"required":["usesShell","shellCommand"]}]},"executableString":{"type":"string","not":{"pattern":"\\$\\{[^}]+\\}"}},"absolutePathString":{"allOf":[{"$ref":"#/$defs/executableString"},{"pattern":"^(?:/|[A-Za-z]:[\\\\/]|\\\\\\\\)"}]},"preflight":{"type":"object","properties":{"status":{"enum":["pending","verified"]},"ran":{"type":"boolean"},"command":{"type":"string"},"exitCode":{"type":["number","null"]},"durationMs":{"type":["number","null"]},"observedTestCount":{"type":["number","null"]},"stdoutSummary":{"type":"string"},"stderrSummary":{"type":"string"},"maxTestCount":{"type":"integer","minimum":1,"maximum":1000}},"additionalProperties":true,"required":["maxTestCount"]},"ciWiring":{"type":"object","required":["status","replayability"],"properties":{"status":{"enum":["pass","fail","skip","unknown"]},"provider":{"type":"string"},"configFile":{"$ref":"#/$defs/absolutePathString"},"workflow":{"type":["string","null"]},"job":{"type":["string","null"]},"step":{"type":["string","null"]},"matrix":{"type":"object"},"runner":{"type":["string","null"]},"shell":{"type":["string","null"],"minLength":1,"pattern":"^[^\\u0000-\\u001F\\u007F-\\u009F\\u00AD\\u0600-\\u0605\\u061C\\u06DD\\u070F\\u0890\\u0891\\u08E2\\u180B-\\u180F\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u206F\\uFE00-\\uFE0F\\uFEFF\\uFFF9-\\uFFFB]*$"},"workingDirectory":{"anyOf":[{"$ref":"#/$defs/absolutePathString"},{"type":"null"}]},"inheritedEnv":{"type":"object"},"requiredSecretEnvVars":{"type":"array","items":{"type":"string"}},"setupCommandIds":{"type":"array","items":{"type":"string"}},"whySelected":{"type":"string"},"workflowEnv":{"type":"object"},"jobEnv":{"type":"object"},"stepEnv":{"type":"object"},"initialization":{"type":"object","required":["status","evidence"],"properties":{"status":{"enum":["configured","not_configured","unknown"]},"evidence":{"type":"array","items":{"type":"string"}}},"allOf":[{"if":{"properties":{"status":{"enum":["configured","not_configured"]}},"required":["status"]},"then":{"properties":{"evidence":{"minItems":1}}}}],"additionalProperties":false},"packageScriptExpansionChain":{"type":"array","items":{"type":"string"}},"runnerToolChain":{"type":"array","items":{"type":"string"}},"unresolved":{"type":"array","items":{"type":"string"}},"diagnosis":{"type":"string"},"reason":{"type":"string"},"replayability":{"enum":["replayable","not_replayable"]},"replayBlocker":{"type":"string","minLength":1},"ciWiringCommand":false},"additionalProperties":true,"allOf":[{"if":{"properties":{"status":{"enum":["skip","unknown"]}}},"then":{"anyOf":[{"required":["diagnosis"]},{"required":["reason"]}]}},{"if":{"properties":{"replayability":{"const":"not_replayable"}},"required":["replayability"]},"then":{"required":["replayBlocker"],"properties":{"status":{"enum":["skip","unknown"]}}}}]},"generatedTestStrategy":{"type":"object","required":["status"],"properties":{"status":{"enum":["planned","verified","proposed","not_possible"]},"reason":{"type":["string","null"]},"adapter":{"enum":["jest","vitest","mocha","cucumber","cypress","playwright","node:test","generic-js","custom","unknown"]},"testDirectory":{"anyOf":[{"$ref":"#/$defs/absolutePathString"},{"type":"null"}]},"moduleSystem":{"enum":["commonjs","esm","typescript","unknown"]},"fileExtension":{"type":"string"},"supportsFocusedSingleFileRun":{"type":"boolean"},"usesMultipleFiles":{"type":"boolean"},"files":{"type":"array","maxItems":8,"items":{"$ref":"#/$defs/generatedFile"}},"scenarios":{"type":"array","items":{"$ref":"#/$defs/scenario"}},"verification":{"$ref":"#/$defs/verification"},"cleanupPaths":{"type":"array","items":{"$ref":"#/$defs/absolutePathString"}},"limitations":{"type":"array","items":{"type":"string"}}},"additionalProperties":true,"allOf":[{"if":{"properties":{"status":{"enum":["planned","verified"]}}},"then":{"required":["files","scenarios","cleanupPaths"],"properties":{"scenarios":{"allOf":[{"items":{"required":["testIdentities","expectedWithoutDatadog"],"properties":{"testIdentities":{"minItems":1}}}},{"contains":{"properties":{"id":{"const":"basic-pass"}},"required":["id"]}},{"contains":{"properties":{"id":{"const":"atr-fail-once"}},"required":["id"]}},{"contains":{"properties":{"id":{"const":"test-management-target"}},"required":["id"]}}]}}}}]},"generatedFile":{"type":"object","required":["path","contentLines"],"properties":{"path":{"$ref":"#/$defs/absolutePathString"},"role":{"enum":["test","feature","steps","support","state","config"]},"contentLines":{"type":"array","maxItems":256,"items":{"type":"string","maxLength":4096,"pattern":"^[^\\r\\n\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u001F\\u007F-\\u009F\\u061C\\u180B-\\u180F\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u206F\\uFE00-\\uFE0F\\uFEFF]*$"}}},"additionalProperties":true},"scenario":{"type":"object","required":["id","runCommand"],"properties":{"id":{"type":"string"},"purpose":{"type":"string"},"runCommand":{"$ref":"#/$defs/command"},"expectedWithoutDatadog":{"$ref":"#/$defs/expectedWithoutDatadog"},"testIdentities":{"type":"array","items":{"$ref":"#/$defs/testIdentity"}}},"additionalProperties":true,"allOf":[{"if":{"required":["id"],"properties":{"id":{"const":"basic-pass"}}},"then":{"properties":{"expectedWithoutDatadog":{"properties":{"exitCode":{"const":0},"observedTestCount":{"const":1}}}}}},{"if":{"required":["id"],"properties":{"id":{"const":"atr-fail-once"}}},"then":{"properties":{"expectedWithoutDatadog":{"properties":{"exitCode":{"const":1},"observedTestCount":{"const":1}}}}}},{"if":{"required":["id"],"properties":{"id":{"const":"test-management-target"}}},"then":{"properties":{"expectedWithoutDatadog":{"properties":{"exitCode":{"const":0},"observedTestCount":{"const":1}}}}}}]},"expectedWithoutDatadog":{"type":"object","required":["exitCode","observedTestCount"],"properties":{"exitCode":{"type":"number"},"observedTestCount":{"type":"number"}},"additionalProperties":true},"testIdentity":{"type":"object","required":["name"],"properties":{"suite":{"type":["string","null"]},"name":{"type":"string","minLength":1},"file":{"anyOf":[{"$ref":"#/$defs/absolutePathString"},{"type":"null"}]},"parameters":{"type":["object","null"]}},"additionalProperties":true},"verification":{"type":"object","properties":{"createdTemporaryFiles":{"type":"boolean"},"ranScenarioIds":{"type":"array","items":{"type":"string"}},"exitCode":{"type":["number","null"]},"durationMs":{"type":["number","null"]},"observedTestCount":{"type":["number","null"]},"cleanupCompleted":{"type":"boolean"},"stdoutSummary":{"type":"string"},"stderrSummary":{"type":"string"}},"additionalProperties":true}}}

package/ci/validate-test-optimization.js

@@ -0,0 +1,3 @@
+'use strict'
+
+require('./test-optimization-validation/cli').main(process.argv.slice(2))

package/ext/exporters.js

@@ -5,6 +5,7 @@ module.exports = {
   AGENTLESS: 'agentless',
   DATADOG: 'datadog',
   AGENT_PROXY: 'agent_proxy',
+  CI_VALIDATION: 'ci_validation',
   JEST_WORKER: 'jest_worker',
   CUCUMBER_WORKER: 'cucumber_worker',
   MOCHA_WORKER: 'mocha_worker',