dd-trace 5.93.0 → 5.95.0

package/packages/dd-trace/src/azure_metadata.js

@@ -5,10 +5,9 @@
 const os = require('os')
 const {
   getEnvironmentVariable,
-  getEnvironmentVariables,
   getValueFromEnvSources,
 } = require('./config/helper')
-const { getIsAzureFunction, getIsFlexConsumptionAzureFunction } = require('./serverless')
+const { getIsAzureFunction } = require('./serverless')
 
 function extractSubscriptionID (ownerName) {
   if (ownerName !== undefined) {
@@ -46,32 +45,33 @@ function trimObject (obj) {
 }
 
 function buildMetadata () {
-  const {
-    COMPUTERNAME,
-    FUNCTIONS_EXTENSION_VERSION,
-    FUNCTIONS_WORKER_RUNTIME,
-    FUNCTIONS_WORKER_RUNTIME_VERSION,
-    WEBSITE_INSTANCE_ID,
-    WEBSITE_OWNER_NAME,
-    WEBSITE_OS,
-    WEBSITE_RESOURCE_GROUP,
-    WEBSITE_SITE_NAME,
-  } = getEnvironmentVariables()
+  const COMPUTERNAME = getEnvironmentVariable('COMPUTERNAME')
+  const FUNCTIONS_EXTENSION_VERSION = getEnvironmentVariable('FUNCTIONS_EXTENSION_VERSION')
+  const FUNCTIONS_WORKER_RUNTIME = getEnvironmentVariable('FUNCTIONS_WORKER_RUNTIME')
+  const FUNCTIONS_WORKER_RUNTIME_VERSION = getEnvironmentVariable('FUNCTIONS_WORKER_RUNTIME_VERSION')
+  const WEBSITE_INSTANCE_ID = getEnvironmentVariable('WEBSITE_INSTANCE_ID')
+  const WEBSITE_OWNER_NAME = getEnvironmentVariable('WEBSITE_OWNER_NAME')
+  const WEBSITE_OS = getEnvironmentVariable('WEBSITE_OS')
+  const WEBSITE_RESOURCE_GROUP = getEnvironmentVariable('WEBSITE_RESOURCE_GROUP')
+  const WEBSITE_SITE_NAME = getEnvironmentVariable('WEBSITE_SITE_NAME')
+  const WEBSITE_SKU = getEnvironmentVariable('WEBSITE_SKU')
 
   const DD_AZURE_RESOURCE_GROUP = getValueFromEnvSources('DD_AZURE_RESOURCE_GROUP')
+  const isAzureFunction = FUNCTIONS_EXTENSION_VERSION !== undefined && FUNCTIONS_WORKER_RUNTIME !== undefined
+  const isFlexConsumptionAzureFunction = isAzureFunction && WEBSITE_SKU === 'FlexConsumption'
 
   const subscriptionID = extractSubscriptionID(WEBSITE_OWNER_NAME)
 
   const siteName = WEBSITE_SITE_NAME
 
-  const [siteKind, siteType] = getIsAzureFunction()
+  const [siteKind, siteType] = isAzureFunction
     ? ['functionapp', 'function']
     : ['app', 'app']
 
   // Azure Functions on Flex Consumption plans require the `DD_AZURE_RESOURCE_GROUP` env var.
   // If this logic ever changes, update the logic in `libdatadog`, `serverless-components/src/datadog-trace-agent`,
   // and the serverless compat layers accordingly.
-  const resourceGroup = getIsFlexConsumptionAzureFunction()
+  const resourceGroup = isFlexConsumptionAzureFunction
     ? (DD_AZURE_RESOURCE_GROUP ?? WEBSITE_RESOURCE_GROUP ?? extractResourceGroup(WEBSITE_OWNER_NAME))
     : (WEBSITE_RESOURCE_GROUP ?? extractResourceGroup(WEBSITE_OWNER_NAME))
 

package/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js

@@ -16,6 +16,41 @@ const {
 } = require('../../ci-visibility/telemetry')
 
 const { getNumFromKnownTests } = require('../../plugins/util/test')
+const { buildCacheKey, writeToCache, withCache } = require('../requests/fs-cache')
+
+const MAX_KNOWN_TESTS_PAGES = 10_000
+
+/**
+ * Deep-merges page tests into aggregate.
+ * Structure: { module: { suite: [testName, ...] } }
+ *
+ * @param {object | null} aggregate
+ * @param {object | null} page
+ * @returns {object | null}
+ */
+function mergeKnownTests (aggregate, page) {
+  if (!page) return aggregate
+  if (!aggregate) return page
+
+  for (const [moduleName, suites] of Object.entries(page)) {
+    if (!suites) continue
+
+    if (!aggregate[moduleName]) {
+      aggregate[moduleName] = suites
+      continue
+    }
+
+    for (const [suiteName, tests] of Object.entries(suites)) {
+      if (!tests || tests.length === 0) continue
+
+      aggregate[moduleName][suiteName] = aggregate[moduleName][suiteName]
+        ? [...aggregate[moduleName][suiteName], ...tests]
+        : tests
+    }
+  }
+
+  return aggregate
+}
 
 function getKnownTests ({
   url,
@@ -32,6 +67,71 @@ function getKnownTests ({
   runtimeName,
   runtimeVersion,
   custom,
+}, done) {
+  const cacheKey = buildCacheKey('known-tests', [
+    sha, service, env, repositoryUrl, osPlatform, osVersion, osArchitecture,
+    runtimeName, runtimeVersion, custom,
+  ])
+
+  withCache(cacheKey, (activeCacheKey, cb) => {
+    fetchFromApi({
+      url,
+      isEvpProxy,
+      evpProxyPrefix,
+      isGzipCompatible,
+      env,
+      service,
+      repositoryUrl,
+      sha,
+      osVersion,
+      osPlatform,
+      osArchitecture,
+      runtimeName,
+      runtimeVersion,
+      custom,
+      cacheKey: activeCacheKey,
+    }, cb)
+  }, done)
+}
+
+/**
+ * Fetches known tests from the API with cursor-based pagination and writes the
+ * result to cache on success.
+ *
+ * @param {object} params
+ * @param {string} params.url
+ * @param {boolean} params.isEvpProxy
+ * @param {string} params.evpProxyPrefix
+ * @param {boolean} params.isGzipCompatible
+ * @param {string} params.env
+ * @param {string} params.service
+ * @param {string} params.repositoryUrl
+ * @param {string} params.sha
+ * @param {string} params.osVersion
+ * @param {string} params.osPlatform
+ * @param {string} params.osArchitecture
+ * @param {string} params.runtimeName
+ * @param {string} params.runtimeVersion
+ * @param {object} [params.custom]
+ * @param {string | null} params.cacheKey
+ * @param {Function} done
+ */
+function fetchFromApi ({
+  url,
+  isEvpProxy,
+  evpProxyPrefix,
+  isGzipCompatible,
+  env,
+  service,
+  repositoryUrl,
+  sha,
+  osVersion,
+  osPlatform,
+  osArchitecture,
+  runtimeName,
+  runtimeVersion,
+  custom,
+  cacheKey,
 }, done) {
   const options = {
     path: '/api/v2/ci/libraries/tests',
@@ -59,53 +159,96 @@ function getKnownTests ({
     options.headers['dd-api-key'] = apiKey
   }
 
-  const data = JSON.stringify({
-    data: {
-      id: id().toString(10),
-      type: 'ci_app_libraries_tests_request',
-      attributes: {
-        configurations: {
-          'os.platform': osPlatform,
-          'os.version': osVersion,
-          'os.architecture': osArchitecture,
-          'runtime.name': runtimeName,
-          'runtime.version': runtimeVersion,
-          custom,
-        },
-        service,
-        env,
-        repository_url: repositoryUrl,
-        sha,
-      },
-    },
-  })
+  const configurations = {
+    'os.platform': osPlatform,
+    'os.version': osVersion,
+    'os.architecture': osArchitecture,
+    'runtime.name': runtimeName,
+    'runtime.version': runtimeVersion,
+    custom,
+  }
 
   incrementCountMetric(TELEMETRY_KNOWN_TESTS)
 
   const startTime = Date.now()
+  let aggregateTests = null
+  let totalResponseBytes = 0
+  let pageNumber = 0
+
+  function fetchPage (pageState) {
+    pageNumber++
+
+    if (pageNumber > MAX_KNOWN_TESTS_PAGES) {
+      log.error('Known tests pagination exceeded maximum of %d pages. Aborting.', MAX_KNOWN_TESTS_PAGES)
+      distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
+      return done(new Error(`Known tests pagination exceeded maximum of ${MAX_KNOWN_TESTS_PAGES} pages`))
+    }
+
+    const pageInfo = pageState ? { page_state: pageState } : {}
+
+    const data = JSON.stringify({
+      data: {
+        id: id().toString(10),
+        type: 'ci_app_libraries_tests_request',
+        attributes: {
+          configurations,
+          service,
+          env,
+          repository_url: repositoryUrl,
+          sha,
+          page_info: pageInfo,
+        },
+      },
+    })
+
+    request(data, options, (err, res, statusCode) => {
+      if (err) {
+        distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
+        incrementCountMetric(TELEMETRY_KNOWN_TESTS_ERRORS, { statusCode })
+        return done(err)
+      }
 
-  request(data, options, (err, res, statusCode) => {
-    distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
-    if (err) {
-      incrementCountMetric(TELEMETRY_KNOWN_TESTS_ERRORS, { statusCode })
-      done(err)
-    } else {
       try {
-        const { data: { attributes: { tests: knownTests } } } = JSON.parse(res)
+        totalResponseBytes += res.length
+
+        const { data: { attributes } } = JSON.parse(res)
+        const { tests: pageTests, page_info: responsePageInfo } = attributes
+
+        aggregateTests = mergeKnownTests(aggregateTests, pageTests)
+
+        // Check if there are more pages
+        if (responsePageInfo && responsePageInfo.has_next) {
+          if (!responsePageInfo.cursor) {
+            log.error(
+              'Known tests response has has_next=true but no cursor on page %d. Aborting pagination.', pageNumber
+            )
+            distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
+            return done(new Error('Known tests pagination: has_next=true but no cursor'))
+          }
+          return fetchPage(responsePageInfo.cursor)
+        }
 
-        const numTests = getNumFromKnownTests(knownTests)
+        // Done — no more pages
+        distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
+
+        const numTests = getNumFromKnownTests(aggregateTests)
 
         distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS, {}, numTests)
-        distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES, {}, res.length)
+        distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES, {}, totalResponseBytes)
+
+        log.debug('Number of received known tests: %d', numTests)
 
-        log.debug('Number of received known tests:', numTests)
+        writeToCache(cacheKey, aggregateTests)
 
-        done(null, knownTests)
+        done(null, aggregateTests)
       } catch (err) {
+        distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
         done(err)
       }
-    }
-  })
+    })
+  }
+
+  fetchPage(null)
 }
 
 module.exports = { getKnownTests }

package/packages/dd-trace/src/ci-visibility/exporters/test-worker/writer.js

@@ -2,10 +2,6 @@
 const { JSONEncoder } = require('../../encode/json-encoder')
 const { getEnvironmentVariable } = require('../../../config/helper')
 const log = require('../../../log')
-const {
-  VITEST_WORKER_TRACE_PAYLOAD_CODE,
-  VITEST_WORKER_LOGS_PAYLOAD_CODE,
-} = require('../../../plugins/util/test')
 
 class Writer {
   constructor (interprocessCode) {
@@ -29,12 +25,6 @@ class Writer {
   }
 
   _sendPayload (data, onDone = () => {}) {
-    // ## Jest
-    // Only available when `child_process` is used for the jest worker.
-    // If worker_threads is used, this will not work
-    // TODO: make `jest` instrumentation compatible with worker_threads
-    // https://github.com/facebook/jest/blob/bb39cb2c617a3334bf18daeca66bd87b7ccab28b/packages/jest-worker/README.md#experimental-worker
-
     // ## Cucumber
     // This reports to the test's main process the same way test data is reported by Cucumber
     // See cucumber code:
@@ -47,19 +37,17 @@ class Writer {
       ? { __tinypool_worker_message__: true, interprocessCode: this._interprocessCode, data }
       : [this._interprocessCode, data]
 
-    const isVitestTestWorker =
-      this._interprocessCode === VITEST_WORKER_TRACE_PAYLOAD_CODE ||
-      this._interprocessCode === VITEST_WORKER_LOGS_PAYLOAD_CODE
-
+    // child_process workers (jest default, cucumber)
     if (process.send) {
       process.send(payload, () => {
         onDone()
       })
-    } else if (isVitestTestWorker) { // TODO: worker_threads are only supported in vitest right now
-      const { isMainThread, parentPort } = require('worker_threads')
-      if (isMainThread) {
-        return onDone()
-      }
+      return
+    }
+
+    // worker_threads (jest --workerThreads, vitest)
+    const { isMainThread, parentPort } = require('node:worker_threads')
+    if (!isMainThread && parentPort) {
       try {
         parentPort.postMessage(payload)
       } catch (error) {
@@ -67,9 +55,10 @@ class Writer {
       } finally {
         onDone()
       }
-    } else {
-      onDone()
+      return
     }
+
+    onDone()
   }
 }
 

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js

@@ -13,6 +13,7 @@ const {
   TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS,
   TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES,
 } = require('../../ci-visibility/telemetry')
+const { buildCacheKey, writeToCache, withCache } = require('../requests/fs-cache')
 
 function getSkippableSuites ({
   url,
@@ -30,6 +31,76 @@ function getSkippableSuites ({
   runtimeVersion,
   custom,
   testLevel = 'suite',
+}, done) {
+  const cacheKey = buildCacheKey('skippable', [
+    sha, service, env, repositoryUrl, osPlatform, osVersion, osArchitecture,
+    runtimeName, runtimeVersion, testLevel, custom,
+  ])
+
+  withCache(cacheKey, (activeCacheKey, cb) => {
+    fetchFromApi({
+      url,
+      isEvpProxy,
+      evpProxyPrefix,
+      isGzipCompatible,
+      env,
+      service,
+      repositoryUrl,
+      sha,
+      osVersion,
+      osPlatform,
+      osArchitecture,
+      runtimeName,
+      runtimeVersion,
+      custom,
+      testLevel,
+      cacheKey: activeCacheKey,
+    }, cb)
+  }, (err, data) => {
+    if (err) return done(err)
+    done(null, data.skippableSuites, data.correlationId)
+  })
+}
+
+/**
+ * Fetches skippable suites from the API and writes the result to cache on success.
+ *
+ * @param {object} params
+ * @param {string} params.url
+ * @param {boolean} params.isEvpProxy
+ * @param {string} params.evpProxyPrefix
+ * @param {boolean} params.isGzipCompatible
+ * @param {string} params.env
+ * @param {string} params.service
+ * @param {string} params.repositoryUrl
+ * @param {string} params.sha
+ * @param {string} params.osVersion
+ * @param {string} params.osPlatform
+ * @param {string} params.osArchitecture
+ * @param {string} params.runtimeName
+ * @param {string} params.runtimeVersion
+ * @param {object} [params.custom]
+ * @param {string} [params.testLevel]
+ * @param {string | null} params.cacheKey
+ * @param {Function} done
+ */
+function fetchFromApi ({
+  url,
+  isEvpProxy,
+  evpProxyPrefix,
+  isGzipCompatible,
+  env,
+  service,
+  repositoryUrl,
+  sha,
+  osVersion,
+  osPlatform,
+  osArchitecture,
+  runtimeName,
+  runtimeVersion,
+  custom,
+  testLevel,
+  cacheKey,
 }, done) {
   const options = {
     path: '/api/v2/ci/tests/skippable',
@@ -109,7 +180,11 @@ function getSkippableSuites ({
         )
         distributionMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES, {}, res.length)
         log.debug('Number of received skippable %ss:', testLevel, skippableSuites.length)
-        done(null, skippableSuites, correlationId)
+
+        const result = { skippableSuites, correlationId }
+        writeToCache(cacheKey, result)
+
+        done(null, result)
       } catch (err) {
         done(err)
       }