dd-trace 5.109.0 → 5.111.0

package/packages/dd-trace/src/ci-visibility/requests/upload-coverage-report.js

@@ -34,9 +34,9 @@ function uploadCoverageReport (
   { filePath, format, testEnvironmentMetadata, url, isEvpProxy, evpProxyPrefix },
   callback
 ) {
-  const apiKey = getConfig().apiKey
+  const { DD_API_KEY } = getConfig()
 
-  if (!apiKey && !isEvpProxy) {
+  if (!DD_API_KEY && !isEvpProxy) {
     return callback(new Error('DD_API_KEY is required for coverage report upload'))
   }
 
@@ -82,7 +82,7 @@ function uploadCoverageReport (
     options.headers['X-Datadog-EVP-Subdomain'] = 'ci-intake'
   } else {
     options.path = '/api/v2/cicovreprt'
-    options.headers['dd-api-key'] = apiKey
+    options.headers['dd-api-key'] = DD_API_KEY
   }
 
   log.debug('Uploading coverage report %s to %s%s', filePath, url, options.path)

package/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js

@@ -16,6 +16,7 @@ const {
 } = require('../telemetry')
 
 const { buildCacheKey, writeToCache, withCache } = require('../requests/fs-cache')
+const { validateTestManagementTestsResponse } = require('../test-optimization-http-cache-schema')
 
 // Calculate the number of tests from the test management tests response, which has a shape like:
 // { module: { suites: { suite: { tests: { testName: { properties: {...} } } } } } }
@@ -39,6 +40,19 @@ function getNumFromTestManagementTests (testManagementTests) {
   return totalNumTests
 }
 
+function parseJsonResponse (rawJson) {
+  return typeof rawJson === 'string' ? JSON.parse(rawJson) : rawJson
+}
+
+function parseTestManagementTestsResponse (rawJson, options = {}) {
+  const parsedResponse = parseJsonResponse(rawJson)
+  if (options.validateRequiredFields) {
+    validateTestManagementTestsResponse(parsedResponse)
+  }
+  const { data: { attributes: { modules: testManagementTests } } } = parsedResponse
+  return testManagementTests
+}
+
 function getTestManagementTests ({
   url,
   isEvpProxy,
@@ -121,12 +135,12 @@ function fetchFromApi ({
     options.path = `${evpProxyPrefix}/api/v2/test/libraries/test-management/tests`
     options.headers['X-Datadog-EVP-Subdomain'] = 'api'
   } else {
-    const { apiKey } = getConfig()
-    if (!apiKey) {
+    const { DD_API_KEY } = getConfig()
+    if (!DD_API_KEY) {
       return done(new Error('Test management tests were not fetched because Datadog API key is not defined.'))
     }
 
-    options.headers['dd-api-key'] = apiKey
+    options.headers['dd-api-key'] = DD_API_KEY
   }
 
   const data = JSON.stringify({
@@ -155,7 +169,7 @@ function fetchFromApi ({
       done(err)
     } else {
       try {
-        const { data: { attributes: { modules: testManagementTests } } } = JSON.parse(res)
+        const testManagementTests = parseTestManagementTestsResponse(res)
 
         const numTests = getNumFromTestManagementTests(testManagementTests)
 
@@ -174,4 +188,8 @@ function fetchFromApi ({
   })
 }
 
-module.exports = { getTestManagementTests }
+module.exports = {
+  getNumFromTestManagementTests, // Exported for later use in the cache
+  getTestManagementTests,
+  parseTestManagementTestsResponse,
+}

package/packages/dd-trace/src/ci-visibility/test-optimization-http-cache-schema.js

@@ -0,0 +1,128 @@
+'use strict'
+
+const REQUIRED_SETTINGS_KEYS = [
+  'code_coverage',
+  'tests_skipping',
+  'itr_enabled',
+  'require_git',
+  'early_flake_detection',
+  'flaky_test_retries_enabled',
+  'di_enabled',
+  'known_tests_enabled',
+  'test_management',
+  'impacted_tests_enabled',
+  'coverage_report_upload_enabled',
+]
+
+function isObject (value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value)
+}
+
+function assertObject (value, message) {
+  if (!isObject(value)) {
+    throw new Error(message)
+  }
+}
+
+function assertHasKeys (value, keys, endpoint) {
+  for (const key of keys) {
+    if (!Object.hasOwn(value, key)) {
+      throw new Error(`Invalid ${endpoint} response: missing ${key}`)
+    }
+  }
+}
+
+function getAttributes (response, endpoint) {
+  const attributes = response?.data?.attributes
+  assertObject(attributes, `Invalid ${endpoint} response: attributes must be an object`)
+  return attributes
+}
+
+function validateSettingsResponse (response) {
+  const attributes = response?.data?.attributes ?? response
+  assertObject(attributes, 'Invalid settings response: attributes must be an object')
+  assertHasKeys(attributes, REQUIRED_SETTINGS_KEYS, 'settings')
+  assertObject(attributes.early_flake_detection, 'Invalid settings response: early_flake_detection must be an object')
+  assertObject(attributes.test_management, 'Invalid settings response: test_management must be an object')
+  assertHasKeys(attributes.early_flake_detection, ['enabled'], 'settings early_flake_detection')
+  assertHasKeys(attributes.test_management, ['enabled'], 'settings test_management')
+}
+
+function validateKnownTestsResponse (response) {
+  const attributes = getAttributes(response, 'known tests')
+  assertHasKeys(attributes, ['tests'], 'known tests')
+
+  const { tests } = attributes
+  if (tests === null) return
+  assertObject(tests, 'Invalid known tests response: tests must be an object or null')
+
+  for (const suites of Object.values(tests)) {
+    assertObject(suites, 'Invalid known tests response: module suites must be objects')
+    for (const testNames of Object.values(suites)) {
+      if (!Array.isArray(testNames)) {
+        throw new TypeError('Invalid known tests response: suite tests must be arrays')
+      }
+      for (const testName of testNames) {
+        if (typeof testName !== 'string') {
+          throw new TypeError('Invalid known tests response: test names must be strings')
+        }
+      }
+    }
+  }
+}
+
+function validateSkippableTestsResponse (response) {
+  assertObject(response, 'Invalid skippable tests response: response must be an object')
+  if (!Array.isArray(response.data)) {
+    throw new TypeError('Invalid skippable tests response: data must be an array')
+  }
+  if (response.meta !== undefined) {
+    assertObject(response.meta, 'Invalid skippable tests response: meta must be an object')
+  }
+  if (response.meta?.coverage !== undefined) {
+    assertObject(response.meta.coverage, 'Invalid skippable tests response: meta.coverage must be an object')
+  }
+  if (response.meta?.correlation_id !== undefined && typeof response.meta.correlation_id !== 'string') {
+    throw new Error('Invalid skippable tests response: meta.correlation_id must be a string')
+  }
+
+  for (const item of response.data) {
+    assertObject(item, 'Invalid skippable tests response: data entries must be objects')
+    if (typeof item.type !== 'string') {
+      throw new TypeError('Invalid skippable tests response: data entry type must be a string')
+    }
+    assertObject(item.attributes, 'Invalid skippable tests response: data entry attributes must be an object')
+    if ((item.type === 'suite' || item.type === 'test') && typeof item.attributes.suite !== 'string') {
+      throw new Error('Invalid skippable tests response: data entry suite must be a string')
+    }
+    if (item.type === 'test' && typeof item.attributes.name !== 'string') {
+      throw new Error('Invalid skippable tests response: data entry name must be a string')
+    }
+  }
+}
+
+function validateTestManagementTestsResponse (response) {
+  const attributes = getAttributes(response, 'test management tests')
+  assertHasKeys(attributes, ['modules'], 'test management tests')
+  assertObject(attributes.modules, 'Invalid test management tests response: modules must be an object')
+
+  for (const testModule of Object.values(attributes.modules)) {
+    assertObject(testModule, 'Invalid test management tests response: modules entries must be objects')
+    assertObject(testModule.suites, 'Invalid test management tests response: module suites must be objects')
+    for (const suite of Object.values(testModule.suites)) {
+      assertObject(suite, 'Invalid test management tests response: suites entries must be objects')
+      assertObject(suite.tests, 'Invalid test management tests response: suite tests must be objects')
+      for (const test of Object.values(suite.tests)) {
+        assertObject(test, 'Invalid test management tests response: tests entries must be objects')
+        assertObject(test.properties, 'Invalid test management tests response: test properties must be objects')
+      }
+    }
+  }
+}
+
+module.exports = {
+  validateKnownTestsResponse,
+  validateSettingsResponse,
+  validateSkippableTestsResponse,
+  validateTestManagementTestsResponse,
+}

package/packages/dd-trace/src/ci-visibility/test-optimization-http-cache.js

@@ -0,0 +1,287 @@
+'use strict'
+
+const fs = require('node:fs')
+const path = require('node:path')
+
+const log = require('../log')
+const { getNumFromKnownTests } = require('../plugins/util/test')
+const { parseKnownTestsResponse } = require('./early-flake-detection/get-known-tests')
+const { parseSkippableSuitesResponse } = require('./intelligent-test-runner/get-skippable-suites')
+const { parseLibraryConfigurationResponse } = require('./requests/get-library-configuration')
+const {
+  getNumFromTestManagementTests,
+  parseTestManagementTestsResponse,
+} = require('./test-management/get-test-management-tests')
+const {
+  incrementCountMetric,
+  distributionMetric,
+  TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE,
+  TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS,
+  TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES,
+  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES,
+  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS,
+  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES,
+  TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_TESTS,
+  TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_BYTES,
+} = require('./telemetry')
+
+const CACHE_MISS = Symbol('test optimization http cache miss')
+
+const PLAN_FOLDER = '.testoptimization'
+const MANIFEST_FILE_NAME = 'manifest.txt'
+const SUPPORTED_MANIFEST_VERSION = '1'
+const CACHE_FOLDER_NAME = 'cache'
+const HTTP_CACHE_FOLDER_NAME = 'http'
+
+const ENV_MANIFEST_FILE = 'TEST_OPTIMIZATION_MANIFEST_FILE'
+const ENV_MANIFEST_FILE_ALIAS = 'DD_TEST_OPTIMIZATION_MANIFEST_FILE'
+const ENV_RUNFILES_DIR = 'RUNFILES_DIR'
+const ENV_RUNFILES_MANIFEST_FILE = 'RUNFILES_MANIFEST_FILE'
+const ENV_TEST_SRCDIR = 'TEST_SRCDIR'
+
+const SETTINGS_FILE_NAME = 'settings.json'
+const KNOWN_TESTS_FILE_NAME = 'known_tests.json'
+const SKIPPABLE_TESTS_FILE_NAME = 'skippable_tests.json'
+const TEST_MANAGEMENT_FILE_NAME = 'test_management.json'
+
+const RUNFILES_MANIFEST_SEPARATOR = ' '
+
+function parseManifestVersion (content) {
+  // Supported just the number version or 'version=x'
+  const version = content.replace(/^\uFEFF/, '').trim()
+  const match = version.match(/^version=(.+)$/)
+  return match ? match[1].trim() : version
+}
+
+class TestOptimizationHttpCache {
+  constructor ({ cwd = process.cwd(), env } = {}) {
+    this._cwd = cwd
+    // This cache intentionally consumes env vars that are not tracer config keys. (??)
+    // eslint-disable-next-line eslint-rules/eslint-process-env
+    this._env = env ?? process.env
+    this._manifestPath = this._resolveManifestPath()
+    this._testOptimizationPath = undefined
+    this._httpCachePath = undefined
+    this._available = false
+
+    this._buildReader()
+  }
+
+  isAvailable () {
+    return this._available
+  }
+
+  readSettings () {
+    const payload = this._readFile(SETTINGS_FILE_NAME)
+    if (payload === CACHE_MISS) {
+      this._disable()
+      return CACHE_MISS
+    }
+
+    try {
+      const settings = parseLibraryConfigurationResponse(payload, undefined, { validateRequiredFields: true })
+      incrementCountMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE, settings)
+      return settings
+    } catch (err) {
+      this._logInvalidCacheFile(SETTINGS_FILE_NAME, err)
+      this._disable()
+      return CACHE_MISS
+    }
+  }
+
+  readKnownTests () {
+    const payload = this._readFile(KNOWN_TESTS_FILE_NAME)
+    if (payload === CACHE_MISS) return CACHE_MISS
+
+    try {
+      const knownTests = parseKnownTestsResponse(payload, { validateRequiredFields: true })
+      distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS, {}, getNumFromKnownTests(knownTests))
+      distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES, {}, payload.length)
+      return knownTests
+    } catch (err) {
+      this._logInvalidCacheFile(KNOWN_TESTS_FILE_NAME, err)
+      return CACHE_MISS
+    }
+  }
+
+  readSkippableSuites (options = {}) {
+    const payload = this._readFile(SKIPPABLE_TESTS_FILE_NAME)
+    if (payload === CACHE_MISS) return CACHE_MISS
+
+    try {
+      const parsedResponse = JSON.parse(payload)
+      const result = parseSkippableSuitesResponse(parsedResponse, { ...options, validateRequiredFields: true })
+      const testLevel = options.testLevel || 'suite'
+      const skippableItems = parsedResponse.data.filter(({ type }) => type === testLevel)
+      incrementCountMetric(
+        testLevel === 'test'
+          ? TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS
+          : TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES,
+        {},
+        skippableItems.length
+      )
+      distributionMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES, {}, payload.length)
+      return result
+    } catch (err) {
+      this._logInvalidCacheFile(SKIPPABLE_TESTS_FILE_NAME, err)
+      return CACHE_MISS
+    }
+  }
+
+  hasValidSkippableSuites (options = {}) {
+    const payload = this._readFile(SKIPPABLE_TESTS_FILE_NAME)
+    if (payload === CACHE_MISS) return false
+
+    try {
+      parseSkippableSuitesResponse(JSON.parse(payload), { ...options, validateRequiredFields: true })
+      return true
+    } catch {
+      return false
+    }
+  }
+
+  readTestManagementTests () {
+    const payload = this._readFile(TEST_MANAGEMENT_FILE_NAME)
+    if (payload === CACHE_MISS) return CACHE_MISS
+
+    try {
+      const testManagementTests = parseTestManagementTestsResponse(payload, { validateRequiredFields: true })
+      distributionMetric(
+        TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_TESTS,
+        {},
+        getNumFromTestManagementTests(testManagementTests)
+      )
+      distributionMetric(TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_BYTES, {}, payload.length)
+      return testManagementTests
+    } catch (err) {
+      this._logInvalidCacheFile(TEST_MANAGEMENT_FILE_NAME, err)
+      return CACHE_MISS
+    }
+  }
+
+  _buildReader () {
+    if (!this._manifestPath) {
+      log.debug('Test Optimization HTTP cache manifest not found')
+      return
+    }
+
+    const version = this._readManifestVersion(this._manifestPath)
+    if (version !== SUPPORTED_MANIFEST_VERSION) {
+      log.debug('Unsupported Test Optimization HTTP cache manifest version %j at %s', version, this._manifestPath)
+      return
+    }
+
+    this._testOptimizationPath = path.dirname(this._manifestPath)
+    this._httpCachePath = path.join(
+      this._testOptimizationPath,
+      CACHE_FOLDER_NAME,
+      HTTP_CACHE_FOLDER_NAME
+    )
+
+    const settingsPath = path.join(this._httpCachePath, SETTINGS_FILE_NAME)
+    if (!fs.existsSync(settingsPath)) {
+      log.debug('Test Optimization HTTP cache settings file not found at %s', settingsPath)
+      return
+    }
+
+    log.debug('Test Optimization HTTP cache found at %s', this._httpCachePath)
+    this._available = true
+  }
+
+  _resolveManifestPath () {
+    const localManifest = path.join(this._cwd, PLAN_FOLDER, MANIFEST_FILE_NAME)
+    if (fs.existsSync(localManifest)) {
+      return localManifest
+    }
+
+    const manifestFile = this._env[ENV_MANIFEST_FILE] || this._env[ENV_MANIFEST_FILE_ALIAS]
+    if (!manifestFile) return
+
+    const resolvedManifestPath = this._resolveRunfilePath(manifestFile)
+    if (fs.existsSync(resolvedManifestPath)) {
+      return resolvedManifestPath
+    }
+  }
+
+  _resolveRunfilePath (manifestFile) {
+    if (fs.existsSync(manifestFile)) {
+      return manifestFile
+    }
+
+    const runfilesDir = this._env[ENV_RUNFILES_DIR]
+    if (runfilesDir) {
+      const candidate = path.join(runfilesDir, manifestFile)
+      if (fs.existsSync(candidate)) {
+        return candidate
+      }
+    }
+
+    const runfilesManifestCandidate = this._resolveRunfilePathFromManifest(manifestFile)
+    if (runfilesManifestCandidate) {
+      return runfilesManifestCandidate
+    }
+
+    const testSrcdir = this._env[ENV_TEST_SRCDIR]
+    if (testSrcdir) {
+      const candidate = path.join(testSrcdir, manifestFile)
+      if (fs.existsSync(candidate)) {
+        return candidate
+      }
+    }
+
+    return manifestFile
+  }
+
+  _resolveRunfilePathFromManifest (manifestFile) {
+    const runfilesManifest = this._env[ENV_RUNFILES_MANIFEST_FILE]
+    if (!runfilesManifest || !fs.existsSync(runfilesManifest)) return
+
+    try {
+      const lines = fs.readFileSync(runfilesManifest, 'utf8').split('\n')
+      for (const line of lines) {
+        const separatorIndex = line.indexOf(RUNFILES_MANIFEST_SEPARATOR)
+        if (separatorIndex <= 0) continue
+        if (line.slice(0, separatorIndex) !== manifestFile) continue
+
+        const resolvedPath = line.slice(separatorIndex + 1).trim()
+        if (resolvedPath) return resolvedPath
+      }
+    } catch (err) {
+      log.debug('Failed to resolve Test Optimization HTTP cache manifest from %s: %s', runfilesManifest, err.message)
+    }
+  }
+
+  _readManifestVersion (manifestPath) {
+    try {
+      return parseManifestVersion(fs.readFileSync(manifestPath, 'utf8'))
+    } catch (err) {
+      log.debug('Failed to read Test Optimization HTTP cache manifest %s: %s', manifestPath, err.message)
+    }
+  }
+
+  _readFile (fileName) {
+    if (!this.isAvailable()) return CACHE_MISS
+
+    const filePath = path.join(this._httpCachePath, fileName)
+    try {
+      log.debug('Reading Test Optimization HTTP cache file %s', filePath)
+      return fs.readFileSync(filePath, 'utf8')
+    } catch (err) {
+      log.debug('Test Optimization HTTP cache file %s could not be read: %s', filePath, err.message)
+      return CACHE_MISS
+    }
+  }
+
+  _logInvalidCacheFile (fileName, err) {
+    log.debug('Test Optimization HTTP cache file %s could not be parsed: %s', fileName, err.message)
+  }
+
+  _disable () {
+    this._available = false
+  }
+}
+
+module.exports = {
+  CACHE_MISS,
+  TestOptimizationHttpCache,
+}

package/packages/dd-trace/src/config/defaults.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const dns = require('dns')
 const util = require('util')
 
 const { DD_MAJOR } = require('../../../../version')
@@ -51,7 +50,6 @@ const defaults = {
   isServiceUserProvided: false,
   plugins: true,
   isCiVisibility: false,
-  lookup: dns.lookup,
   logger: undefined,
 }
 
@@ -218,11 +216,20 @@ for (const [canonicalName, entries] of Object.entries(supportedConfigurations))
     )
   }
   for (const entry of entries) {
+    // A deprecated entry that only aliases a canonical option must not surface as
+    // its own Config property/default: the canonical entry already owns the value
+    // and the alias is resolved by helper.js. helper.js drops these from the shared
+    // supported-configurations object (after registering the deprecation), but that
+    // mutation is order-dependent on which module loads first. Skip here too so
+    // `defaults` is identical regardless of load order (it differed in v5, where
+    // major-overrides keeps these entries instead of deleting them outright).
+    if (entry.deprecated && entry.aliases) continue
     if (entry.sensitive) {
       sensitiveConfigurations.add(canonicalName)
     }
-    const configurationNames = entry.internalPropertyName ? [entry.internalPropertyName] : entry.configurationNames
-    const fullPropertyName = configurationNames?.[0] ?? canonicalName
+    const fullPropertyName = entry.namespace
+      ? `${entry.namespace}.${canonicalName}`
+      : (entry.internalPropertyName ?? entry.configurationNames?.[0] ?? canonicalName)
     const type = entry.type.toUpperCase()
 
     let transformer = transformers[entry.transform]
@@ -285,7 +292,7 @@ for (const [canonicalName, entries] of Object.entries(supportedConfigurations))
 
 // Replace the alias with the canonical property name.
 for (const [fullPropertyName, alias] of fallbackConfigurations) {
-  if (configurationsTable[alias].property) {
+  if (configurationsTable[alias]?.property) {
     fallbackConfigurations.set(fullPropertyName, configurationsTable[alias].property)
   }
 }
@@ -347,3 +354,16 @@ module.exports = {
 
   generateTelemetry,
 }
+
+// `dns` is instrumented, so requiring it pulls in the dns plugin, which loads
+// `log`, whose bootstrap calls back into `require('./defaults')`. Resolve the
+// `lookup` default last — after the exports above are fully built — so that the
+// re-entrant require during that cascade sees the complete module instead of a
+// half-initialized one.
+defaults.lookup = require('dns').lookup
+configWithOrigin.set('lookupdefault', {
+  name: 'lookup',
+  value: defaults.lookup,
+  origin: 'default',
+  seq_id: seqId++,
+})