dd-trace 5.110.0 → 5.112.0

package/packages/dd-trace/src/ci-visibility/test-optimization-http-cache-schema.js

@@ -0,0 +1,128 @@
+'use strict'
+
+const REQUIRED_SETTINGS_KEYS = [
+  'code_coverage',
+  'tests_skipping',
+  'itr_enabled',
+  'require_git',
+  'early_flake_detection',
+  'flaky_test_retries_enabled',
+  'di_enabled',
+  'known_tests_enabled',
+  'test_management',
+  'impacted_tests_enabled',
+  'coverage_report_upload_enabled',
+]
+
+function isObject (value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value)
+}
+
+function assertObject (value, message) {
+  if (!isObject(value)) {
+    throw new Error(message)
+  }
+}
+
+function assertHasKeys (value, keys, endpoint) {
+  for (const key of keys) {
+    if (!Object.hasOwn(value, key)) {
+      throw new Error(`Invalid ${endpoint} response: missing ${key}`)
+    }
+  }
+}
+
+function getAttributes (response, endpoint) {
+  const attributes = response?.data?.attributes
+  assertObject(attributes, `Invalid ${endpoint} response: attributes must be an object`)
+  return attributes
+}
+
+function validateSettingsResponse (response) {
+  const attributes = response?.data?.attributes ?? response
+  assertObject(attributes, 'Invalid settings response: attributes must be an object')
+  assertHasKeys(attributes, REQUIRED_SETTINGS_KEYS, 'settings')
+  assertObject(attributes.early_flake_detection, 'Invalid settings response: early_flake_detection must be an object')
+  assertObject(attributes.test_management, 'Invalid settings response: test_management must be an object')
+  assertHasKeys(attributes.early_flake_detection, ['enabled'], 'settings early_flake_detection')
+  assertHasKeys(attributes.test_management, ['enabled'], 'settings test_management')
+}
+
+function validateKnownTestsResponse (response) {
+  const attributes = getAttributes(response, 'known tests')
+  assertHasKeys(attributes, ['tests'], 'known tests')
+
+  const { tests } = attributes
+  if (tests === null) return
+  assertObject(tests, 'Invalid known tests response: tests must be an object or null')
+
+  for (const suites of Object.values(tests)) {
+    assertObject(suites, 'Invalid known tests response: module suites must be objects')
+    for (const testNames of Object.values(suites)) {
+      if (!Array.isArray(testNames)) {
+        throw new TypeError('Invalid known tests response: suite tests must be arrays')
+      }
+      for (const testName of testNames) {
+        if (typeof testName !== 'string') {
+          throw new TypeError('Invalid known tests response: test names must be strings')
+        }
+      }
+    }
+  }
+}
+
+function validateSkippableTestsResponse (response) {
+  assertObject(response, 'Invalid skippable tests response: response must be an object')
+  if (!Array.isArray(response.data)) {
+    throw new TypeError('Invalid skippable tests response: data must be an array')
+  }
+  if (response.meta !== undefined) {
+    assertObject(response.meta, 'Invalid skippable tests response: meta must be an object')
+  }
+  if (response.meta?.coverage !== undefined) {
+    assertObject(response.meta.coverage, 'Invalid skippable tests response: meta.coverage must be an object')
+  }
+  if (response.meta?.correlation_id !== undefined && typeof response.meta.correlation_id !== 'string') {
+    throw new Error('Invalid skippable tests response: meta.correlation_id must be a string')
+  }
+
+  for (const item of response.data) {
+    assertObject(item, 'Invalid skippable tests response: data entries must be objects')
+    if (typeof item.type !== 'string') {
+      throw new TypeError('Invalid skippable tests response: data entry type must be a string')
+    }
+    assertObject(item.attributes, 'Invalid skippable tests response: data entry attributes must be an object')
+    if ((item.type === 'suite' || item.type === 'test') && typeof item.attributes.suite !== 'string') {
+      throw new Error('Invalid skippable tests response: data entry suite must be a string')
+    }
+    if (item.type === 'test' && typeof item.attributes.name !== 'string') {
+      throw new Error('Invalid skippable tests response: data entry name must be a string')
+    }
+  }
+}
+
+function validateTestManagementTestsResponse (response) {
+  const attributes = getAttributes(response, 'test management tests')
+  assertHasKeys(attributes, ['modules'], 'test management tests')
+  assertObject(attributes.modules, 'Invalid test management tests response: modules must be an object')
+
+  for (const testModule of Object.values(attributes.modules)) {
+    assertObject(testModule, 'Invalid test management tests response: modules entries must be objects')
+    assertObject(testModule.suites, 'Invalid test management tests response: module suites must be objects')
+    for (const suite of Object.values(testModule.suites)) {
+      assertObject(suite, 'Invalid test management tests response: suites entries must be objects')
+      assertObject(suite.tests, 'Invalid test management tests response: suite tests must be objects')
+      for (const test of Object.values(suite.tests)) {
+        assertObject(test, 'Invalid test management tests response: tests entries must be objects')
+        assertObject(test.properties, 'Invalid test management tests response: test properties must be objects')
+      }
+    }
+  }
+}
+
+module.exports = {
+  validateKnownTestsResponse,
+  validateSettingsResponse,
+  validateSkippableTestsResponse,
+  validateTestManagementTestsResponse,
+}

package/packages/dd-trace/src/ci-visibility/test-optimization-http-cache.js

@@ -0,0 +1,287 @@
+'use strict'
+
+const fs = require('node:fs')
+const path = require('node:path')
+
+const log = require('../log')
+const { getNumFromKnownTests } = require('../plugins/util/test')
+const { parseKnownTestsResponse } = require('./early-flake-detection/get-known-tests')
+const { parseSkippableSuitesResponse } = require('./intelligent-test-runner/get-skippable-suites')
+const { parseLibraryConfigurationResponse } = require('./requests/get-library-configuration')
+const {
+  getNumFromTestManagementTests,
+  parseTestManagementTestsResponse,
+} = require('./test-management/get-test-management-tests')
+const {
+  incrementCountMetric,
+  distributionMetric,
+  TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE,
+  TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS,
+  TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES,
+  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES,
+  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS,
+  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES,
+  TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_TESTS,
+  TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_BYTES,
+} = require('./telemetry')
+
+const CACHE_MISS = Symbol('test optimization http cache miss')
+
+const PLAN_FOLDER = '.testoptimization'
+const MANIFEST_FILE_NAME = 'manifest.txt'
+const SUPPORTED_MANIFEST_VERSION = '1'
+const CACHE_FOLDER_NAME = 'cache'
+const HTTP_CACHE_FOLDER_NAME = 'http'
+
+const ENV_MANIFEST_FILE = 'TEST_OPTIMIZATION_MANIFEST_FILE'
+const ENV_MANIFEST_FILE_ALIAS = 'DD_TEST_OPTIMIZATION_MANIFEST_FILE'
+const ENV_RUNFILES_DIR = 'RUNFILES_DIR'
+const ENV_RUNFILES_MANIFEST_FILE = 'RUNFILES_MANIFEST_FILE'
+const ENV_TEST_SRCDIR = 'TEST_SRCDIR'
+
+const SETTINGS_FILE_NAME = 'settings.json'
+const KNOWN_TESTS_FILE_NAME = 'known_tests.json'
+const SKIPPABLE_TESTS_FILE_NAME = 'skippable_tests.json'
+const TEST_MANAGEMENT_FILE_NAME = 'test_management.json'
+
+const RUNFILES_MANIFEST_SEPARATOR = ' '
+
+function parseManifestVersion (content) {
+  // Supported just the number version or 'version=x'
+  const version = content.replace(/^\uFEFF/, '').trim()
+  const match = version.match(/^version=(.+)$/)
+  return match ? match[1].trim() : version
+}
+
+class TestOptimizationHttpCache {
+  constructor ({ cwd = process.cwd(), env } = {}) {
+    this._cwd = cwd
+    // This cache intentionally consumes env vars that are not tracer config keys. (??)
+    // eslint-disable-next-line eslint-rules/eslint-process-env
+    this._env = env ?? process.env
+    this._manifestPath = this._resolveManifestPath()
+    this._testOptimizationPath = undefined
+    this._httpCachePath = undefined
+    this._available = false
+
+    this._buildReader()
+  }
+
+  isAvailable () {
+    return this._available
+  }
+
+  readSettings () {
+    const payload = this._readFile(SETTINGS_FILE_NAME)
+    if (payload === CACHE_MISS) {
+      this._disable()
+      return CACHE_MISS
+    }
+
+    try {
+      const settings = parseLibraryConfigurationResponse(payload, undefined, { validateRequiredFields: true })
+      incrementCountMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE, settings)
+      return settings
+    } catch (err) {
+      this._logInvalidCacheFile(SETTINGS_FILE_NAME, err)
+      this._disable()
+      return CACHE_MISS
+    }
+  }
+
+  readKnownTests () {
+    const payload = this._readFile(KNOWN_TESTS_FILE_NAME)
+    if (payload === CACHE_MISS) return CACHE_MISS
+
+    try {
+      const knownTests = parseKnownTestsResponse(payload, { validateRequiredFields: true })
+      distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS, {}, getNumFromKnownTests(knownTests))
+      distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES, {}, payload.length)
+      return knownTests
+    } catch (err) {
+      this._logInvalidCacheFile(KNOWN_TESTS_FILE_NAME, err)
+      return CACHE_MISS
+    }
+  }
+
+  readSkippableSuites (options = {}) {
+    const payload = this._readFile(SKIPPABLE_TESTS_FILE_NAME)
+    if (payload === CACHE_MISS) return CACHE_MISS
+
+    try {
+      const parsedResponse = JSON.parse(payload)
+      const result = parseSkippableSuitesResponse(parsedResponse, { ...options, validateRequiredFields: true })
+      const testLevel = options.testLevel || 'suite'
+      const skippableItems = parsedResponse.data.filter(({ type }) => type === testLevel)
+      incrementCountMetric(
+        testLevel === 'test'
+          ? TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS
+          : TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES,
+        {},
+        skippableItems.length
+      )
+      distributionMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES, {}, payload.length)
+      return result
+    } catch (err) {
+      this._logInvalidCacheFile(SKIPPABLE_TESTS_FILE_NAME, err)
+      return CACHE_MISS
+    }
+  }
+
+  hasValidSkippableSuites (options = {}) {
+    const payload = this._readFile(SKIPPABLE_TESTS_FILE_NAME)
+    if (payload === CACHE_MISS) return false
+
+    try {
+      parseSkippableSuitesResponse(JSON.parse(payload), { ...options, validateRequiredFields: true })
+      return true
+    } catch {
+      return false
+    }
+  }
+
+  readTestManagementTests () {
+    const payload = this._readFile(TEST_MANAGEMENT_FILE_NAME)
+    if (payload === CACHE_MISS) return CACHE_MISS
+
+    try {
+      const testManagementTests = parseTestManagementTestsResponse(payload, { validateRequiredFields: true })
+      distributionMetric(
+        TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_TESTS,
+        {},
+        getNumFromTestManagementTests(testManagementTests)
+      )
+      distributionMetric(TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_BYTES, {}, payload.length)
+      return testManagementTests
+    } catch (err) {
+      this._logInvalidCacheFile(TEST_MANAGEMENT_FILE_NAME, err)
+      return CACHE_MISS
+    }
+  }
+
+  _buildReader () {
+    if (!this._manifestPath) {
+      log.debug('Test Optimization HTTP cache manifest not found')
+      return
+    }
+
+    const version = this._readManifestVersion(this._manifestPath)
+    if (version !== SUPPORTED_MANIFEST_VERSION) {
+      log.debug('Unsupported Test Optimization HTTP cache manifest version %j at %s', version, this._manifestPath)
+      return
+    }
+
+    this._testOptimizationPath = path.dirname(this._manifestPath)
+    this._httpCachePath = path.join(
+      this._testOptimizationPath,
+      CACHE_FOLDER_NAME,
+      HTTP_CACHE_FOLDER_NAME
+    )
+
+    const settingsPath = path.join(this._httpCachePath, SETTINGS_FILE_NAME)
+    if (!fs.existsSync(settingsPath)) {
+      log.debug('Test Optimization HTTP cache settings file not found at %s', settingsPath)
+      return
+    }
+
+    log.debug('Test Optimization HTTP cache found at %s', this._httpCachePath)
+    this._available = true
+  }
+
+  _resolveManifestPath () {
+    const localManifest = path.join(this._cwd, PLAN_FOLDER, MANIFEST_FILE_NAME)
+    if (fs.existsSync(localManifest)) {
+      return localManifest
+    }
+
+    const manifestFile = this._env[ENV_MANIFEST_FILE] || this._env[ENV_MANIFEST_FILE_ALIAS]
+    if (!manifestFile) return
+
+    const resolvedManifestPath = this._resolveRunfilePath(manifestFile)
+    if (fs.existsSync(resolvedManifestPath)) {
+      return resolvedManifestPath
+    }
+  }
+
+  _resolveRunfilePath (manifestFile) {
+    if (fs.existsSync(manifestFile)) {
+      return manifestFile
+    }
+
+    const runfilesDir = this._env[ENV_RUNFILES_DIR]
+    if (runfilesDir) {
+      const candidate = path.join(runfilesDir, manifestFile)
+      if (fs.existsSync(candidate)) {
+        return candidate
+      }
+    }
+
+    const runfilesManifestCandidate = this._resolveRunfilePathFromManifest(manifestFile)
+    if (runfilesManifestCandidate) {
+      return runfilesManifestCandidate
+    }
+
+    const testSrcdir = this._env[ENV_TEST_SRCDIR]
+    if (testSrcdir) {
+      const candidate = path.join(testSrcdir, manifestFile)
+      if (fs.existsSync(candidate)) {
+        return candidate
+      }
+    }
+
+    return manifestFile
+  }
+
+  _resolveRunfilePathFromManifest (manifestFile) {
+    const runfilesManifest = this._env[ENV_RUNFILES_MANIFEST_FILE]
+    if (!runfilesManifest || !fs.existsSync(runfilesManifest)) return
+
+    try {
+      const lines = fs.readFileSync(runfilesManifest, 'utf8').split('\n')
+      for (const line of lines) {
+        const separatorIndex = line.indexOf(RUNFILES_MANIFEST_SEPARATOR)
+        if (separatorIndex <= 0) continue
+        if (line.slice(0, separatorIndex) !== manifestFile) continue
+
+        const resolvedPath = line.slice(separatorIndex + 1).trim()
+        if (resolvedPath) return resolvedPath
+      }
+    } catch (err) {
+      log.debug('Failed to resolve Test Optimization HTTP cache manifest from %s: %s', runfilesManifest, err.message)
+    }
+  }
+
+  _readManifestVersion (manifestPath) {
+    try {
+      return parseManifestVersion(fs.readFileSync(manifestPath, 'utf8'))
+    } catch (err) {
+      log.debug('Failed to read Test Optimization HTTP cache manifest %s: %s', manifestPath, err.message)
+    }
+  }
+
+  _readFile (fileName) {
+    if (!this.isAvailable()) return CACHE_MISS
+
+    const filePath = path.join(this._httpCachePath, fileName)
+    try {
+      log.debug('Reading Test Optimization HTTP cache file %s', filePath)
+      return fs.readFileSync(filePath, 'utf8')
+    } catch (err) {
+      log.debug('Test Optimization HTTP cache file %s could not be read: %s', filePath, err.message)
+      return CACHE_MISS
+    }
+  }
+
+  _logInvalidCacheFile (fileName, err) {
+    log.debug('Test Optimization HTTP cache file %s could not be parsed: %s', fileName, err.message)
+  }
+
+  _disable () {
+    this._available = false
+  }
+}
+
+module.exports = {
+  CACHE_MISS,
+  TestOptimizationHttpCache,
+}

package/packages/dd-trace/src/config/defaults.js

@@ -227,8 +227,9 @@ for (const [canonicalName, entries] of Object.entries(supportedConfigurations))
     if (entry.sensitive) {
       sensitiveConfigurations.add(canonicalName)
     }
-    const configurationNames = entry.internalPropertyName ? [entry.internalPropertyName] : entry.configurationNames
-    const fullPropertyName = configurationNames?.[0] ?? canonicalName
+    const fullPropertyName = entry.namespace
+      ? `${entry.namespace}.${canonicalName}`
+      : (entry.internalPropertyName ?? entry.configurationNames?.[0] ?? canonicalName)
     const type = entry.type.toUpperCase()
 
     let transformer = transformers[entry.transform]