dd-trace 5.66.0 → 5.68.0

package/packages/dd-trace/src/plugins/ci_plugin.js

@@ -36,6 +36,7 @@ const {
   getModifiedTestsFromDiff,
   getPullRequestBaseBranch
 } = require('./util/test')
+const { getRepositoryRoot } = require('./util/git')
 const Plugin = require('./plugin')
 const { COMPONENT } = require('../constants')
 const log = require('../log')
@@ -61,6 +62,7 @@ const {
 const { OS_VERSION, OS_PLATFORM, OS_ARCHITECTURE, RUNTIME_NAME, RUNTIME_VERSION } = require('./util/env')
 const getDiClient = require('../ci-visibility/dynamic-instrumentation')
 const { DD_MAJOR } = require('../../../../version')
+const id = require('../id')
 
 const FRAMEWORK_TO_TRIMMED_COMMAND = {
   vitest: 'vitest run',
@@ -70,12 +72,44 @@ const FRAMEWORK_TO_TRIMMED_COMMAND = {
   jest: 'jest'
 }
 
+const WORKER_EXPORTER_TO_TEST_FRAMEWORK = {
+  vitest_worker: 'vitest',
+  jest_worker: 'jest',
+  cucumber_worker: 'cucumber',
+  mocha_worker: 'mocha',
+  playwright_worker: 'playwright'
+}
+
+const TEST_FRAMEWORKS_TO_SKIP_GIT_METADATA_EXTRACTION = new Set([
+  'vitest',
+  'jest',
+  'mocha',
+  'cucumber',
+])
+
+function getTestSuiteLevelVisibilityTags (testSuiteSpan, testFramework) {
+  const testSuiteSpanContext = testSuiteSpan.context()
+
+  const suiteTags = {
+    [TEST_SUITE_ID]: testSuiteSpanContext.toSpanId(),
+    [TEST_SESSION_ID]: testSuiteSpanContext.toTraceId(),
+    [TEST_COMMAND]: testSuiteSpanContext._tags[TEST_COMMAND],
+    [TEST_MODULE]: testFramework
+  }
+
+  if (testSuiteSpanContext._parentId) {
+    suiteTags[TEST_MODULE_ID] = testSuiteSpanContext._parentId.toString(10)
+  }
+  return suiteTags
+}
+
 module.exports = class CiPlugin extends Plugin {
   constructor (...args) {
     super(...args)
 
     this.fileLineToProbeId = new Map()
     this.rootDir = process.cwd() // fallback in case :session:start events are not emitted
+    this._testSuiteSpansByTestSuite = new Map()
 
     this.addSub(`ci:${this.constructor.id}:library-configuration`, (ctx) => {
       const { onDone, isParallel, frameworkVersion } = ctx
@@ -265,6 +299,53 @@ module.exports = class CiPlugin extends Plugin {
       // TODO: Add telemetry for this type of error
       return onDone({ err: new Error('No modified tests could have been retrieved') })
     })
+
+    this.addSub(`ci:${this.constructor.id}:worker-report:trace`, traces => {
+      const formattedTraces = JSON.parse(traces)
+
+      for (const trace of formattedTraces) {
+        for (const span of trace) {
+          span.span_id = id(span.span_id)
+          span.trace_id = id(span.trace_id)
+          span.parent_id = id(span.parent_id)
+
+          if (span.name?.startsWith(`${this.constructor.id}.`)) {
+            // augment with git information (since it will not be available in the worker)
+            for (const key in this.testEnvironmentMetadata) {
+              // CAREFUL: this bypasses the metadata/metrics distinction
+              // Be careful not to pass numbers in `meta`
+              if (key.startsWith('git.')) {
+                span.meta[key] = this.testEnvironmentMetadata[key]
+              }
+            }
+          }
+
+          // Only test hooks run in the cucumber worker, so the test events do not have the
+          // test session, test module and test suite ids. We have to update them here.
+          if (span.name === 'cucumber.test' || span.name === 'mocha.test') {
+            const testSuite = span.meta[TEST_SUITE]
+            const testSuiteSpan = this._testSuiteSpansByTestSuite.get(testSuite)
+            if (!testSuiteSpan) {
+              log.warn(`Test suite span not found for test span with test suite ${testSuite}`)
+              continue
+            }
+
+            const testSuiteTags = getTestSuiteLevelVisibilityTags(testSuiteSpan, this.constructor.id)
+            span.meta = {
+              ...span.meta,
+              ...testSuiteTags
+            }
+          }
+        }
+        this.tracer._exporter.export(trace)
+      }
+    })
+
+    this.addSub(`ci:${this.constructor.id}:worker-report:logs`, (logsPayloads) => {
+      JSON.parse(logsPayloads).forEach(({ logMessage }) => {
+        this.tracer._exporter.exportDiLogs(this.testEnvironmentMetadata, logMessage)
+      })
+    })
   }
 
   get telemetry () {
@@ -298,7 +379,20 @@ module.exports = class CiPlugin extends Plugin {
       this.di = getDiClient()
     }
 
-    this.testEnvironmentMetadata = getTestEnvironmentMetadata(this.constructor.id, this.config)
+    if (this.testConfiguration) { // no need to recalculate as it's constant
+      return
+    }
+
+    const exporter = this.config.experimental?.exporter
+    const workerTestFramework = WORKER_EXPORTER_TO_TEST_FRAMEWORK[exporter]
+    this.shouldSkipGitMetadataExtraction = workerTestFramework &&
+      TEST_FRAMEWORKS_TO_SKIP_GIT_METADATA_EXTRACTION.has(workerTestFramework)
+
+    this.testEnvironmentMetadata = getTestEnvironmentMetadata(
+      this.constructor.id,
+      this.config,
+      this.shouldSkipGitMetadataExtraction
+    )
 
     const {
       [GIT_REPOSITORY_URL]: repositoryUrl,
@@ -318,9 +412,9 @@ module.exports = class CiPlugin extends Plugin {
       [GIT_COMMIT_HEAD_MESSAGE]: commitHeadMessage
     } = this.testEnvironmentMetadata
 
-    this.repositoryRoot = repositoryRoot || process.cwd()
+    this.repositoryRoot = repositoryRoot || getRepositoryRoot() || process.cwd()
 
-    this.codeOwnersEntries = getCodeOwnersFileEntries(repositoryRoot)
+    this.codeOwnersEntries = getCodeOwnersFileEntries(this.repositoryRoot)
 
     this.ciProviderName = ciProviderName
 

package/packages/dd-trace/src/plugins/index.js

@@ -13,6 +13,7 @@ module.exports = {
   get '@google-cloud/vertexai' () { return require('../../../datadog-plugin-google-cloud-vertexai/src') },
   get '@grpc/grpc-js' () { return require('../../../datadog-plugin-grpc/src') },
   get '@hapi/hapi' () { return require('../../../datadog-plugin-hapi/src') },
+  get '@happy-dom/jest-environment' () { return require('../../../datadog-plugin-jest/src') },
   get '@jest/core' () { return require('../../../datadog-plugin-jest/src') },
   get '@jest/test-sequencer' () { return require('../../../datadog-plugin-jest/src') },
   get '@jest/transform' () { return require('../../../datadog-plugin-jest/src') },
@@ -73,6 +74,7 @@ module.exports = {
   get 'mocha-each' () { return require('../../../datadog-plugin-mocha/src') },
   get vitest () { return require('../../../datadog-plugin-vitest/src') },
   get workerpool () { return require('../../../datadog-plugin-mocha/src') },
+  get tinypool () { return require('../../../datadog-plugin-vitest/src') },
   get moleculer () { return require('../../../datadog-plugin-moleculer/src') },
   get mongodb () { return require('../../../datadog-plugin-mongodb-core/src') },
   get 'mongodb-core' () { return require('../../../datadog-plugin-mongodb-core/src') },

package/packages/dd-trace/src/plugins/util/git-cache.js

@@ -0,0 +1,129 @@
+'use strict'
+
+const os = require('os')
+const path = require('path')
+const fs = require('fs')
+const crypto = require('crypto')
+const cp = require('child_process')
+
+const log = require('../../log')
+const { getEnvironmentVariable } = require('../../config-helper')
+const { isTrue } = require('../../util')
+
+let isGitEnabled = isTrue(getEnvironmentVariable('DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_ENABLED'))
+const GIT_CACHE_DIR = getEnvironmentVariable('DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_DIR') ||
+  path.join(os.tmpdir(), 'dd-trace-git-cache')
+
+function ensureCacheDir () {
+  if (!isGitEnabled) return false
+
+  try {
+    if (fs.existsSync(GIT_CACHE_DIR)) {
+      const stats = fs.statSync(GIT_CACHE_DIR)
+      if (!stats.isDirectory()) {
+        throw new Error(`Cache directory path exists but is not a directory: ${GIT_CACHE_DIR}`)
+      }
+    } else {
+      fs.mkdirSync(GIT_CACHE_DIR, { recursive: true })
+    }
+    return true
+  } catch (err) {
+    log.error('Failed to create git cache directory, disabling cache', err)
+    isGitEnabled = false
+    return false
+  }
+}
+
+// Initialize cache directory at module load time
+ensureCacheDir()
+
+function getCacheKey (cmd, flags) {
+  // Create a hash of the command and flags to use as cache key
+  const commandString = `${cmd} ${flags.join(' ')}`
+  return crypto.createHash('sha256').update(commandString).digest('hex')
+}
+
+function getCacheFilePath (cacheKey) {
+  return path.join(GIT_CACHE_DIR, `${cacheKey}.cache`)
+}
+
+function getCache (cacheKey) {
+  if (!isGitEnabled) return null
+
+  try {
+    const cacheFilePath = getCacheFilePath(cacheKey)
+    if (!fs.existsSync(cacheFilePath)) {
+      return null
+    }
+
+    const content = fs.readFileSync(cacheFilePath, 'utf8')
+    return content
+  } catch (err) {
+    log.error('Failed to read git cache', err)
+    return null
+  }
+}
+
+function setCache (cacheKey, result) {
+  if (!isGitEnabled) return
+
+  // Ensure cache directory exists
+  if (!ensureCacheDir()) return
+
+  try {
+    const cacheFilePath = getCacheFilePath(cacheKey)
+    fs.writeFileSync(cacheFilePath, result, 'utf8')
+  } catch (err) {
+    log.error('Failed to write git cache', err)
+  }
+}
+
+function cachedExec (cmd, flags, options) {
+  if (options === undefined) {
+    options = { stdio: 'pipe' }
+  }
+  if (!isGitEnabled) {
+    return cp.execFileSync(cmd, flags, options)
+  }
+  const cacheKey = getCacheKey(cmd, flags)
+  const cachedResult = getCache(cacheKey)
+  if (cachedResult !== null) {
+    if (cachedResult.startsWith('__GIT_COMMAND_FAILED__')) {
+      let error
+      try {
+        const errorData = cachedResult.replace('__GIT_COMMAND_FAILED__', '')
+        const { message, code, status, errno } = JSON.parse(errorData)
+        error = new Error(message)
+        error.code = code
+        error.status = status
+        error.errno = errno
+      } catch {
+        // we couldn't parse the error data, so we'll throw a generic error
+        throw new Error('Git command failed')
+      }
+      throw error
+    }
+    return cachedResult
+  }
+  try {
+    const result = cp.execFileSync(cmd, flags, options)
+    setCache(cacheKey, result)
+    return result
+  } catch (err) {
+    const cacheValue = '__GIT_COMMAND_FAILED__' +
+      JSON.stringify({
+        code: err.code,
+        status: err.status,
+        errno: err.errno,
+        message: err.message
+      })
+    setCache(cacheKey, cacheValue)
+    throw err
+  }
+}
+
+module.exports = {
+  getCacheKey,
+  getCacheFilePath,
+  cachedExec
+}

package/packages/dd-trace/src/plugins/util/git.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const cp = require('child_process')
 const os = require('os')
 const path = require('path')
 const fs = require('fs')
@@ -36,6 +35,7 @@ const {
 } = require('../../ci-visibility/telemetry')
 const { filterSensitiveInfoFromRepository } = require('./url')
 const { storage } = require('../../../../datadog-core')
+const { cachedExec } = require('./git-cache')
 
 const GIT_REV_LIST_MAX_BUFFER = 12 * 1024 * 1024 // 12MB
 
@@ -58,10 +58,12 @@ function sanitizedExec (
     startTime = Date.now()
   }
   try {
-    let result = cp.execFileSync(cmd, flags, { stdio: 'pipe' }).toString()
+    let result = cachedExec(cmd, flags, { stdio: 'pipe' }).toString()
+
     if (shouldTrim) {
       result = result.replaceAll(/(\r\n|\n|\r)/gm, '')
     }
+
     if (durationMetric) {
       distributionMetric(durationMetric.name, durationMetric.tags, Date.now() - startTime)
     }
@@ -94,7 +96,7 @@ function isGitAvailable () {
   const isWindows = os.platform() === 'win32'
   const command = isWindows ? 'where' : 'which'
   try {
-    cp.execFileSync(command, ['git'], { stdio: 'pipe' })
+    cachedExec(command, ['git'])
     return true
   } catch {
     incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'check_git', exitCode: 'missing' })
@@ -150,27 +152,32 @@ function unshallowRepository (parentOnly = false) {
 
   incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'unshallow' })
   const start = Date.now()
+  let flags = [
+    ...baseGitOptions,
+    revParseHead
+  ]
   try {
-    cp.execFileSync('git', [
-      ...baseGitOptions,
-      revParseHead
-    ], { stdio: 'pipe' })
+    cachedExec('git', flags)
   } catch (err) {
     // If the local HEAD is a commit that has not been pushed to the remote, the above command will fail.
-    log.error('Git plugin error executing git command', err)
+    log.warn(`Git unshallow failed: ${flags.join(' ')}`)
     incrementCountMetric(
       TELEMETRY_GIT_COMMAND_ERRORS,
       { command: 'unshallow', errorType: err.code, exitCode: err.status || err.errno }
     )
-    const upstreamRemote = sanitizedExec('git', ['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{upstream}'])
+    const upstreamRemote = sanitizedExec(
+      'git',
+      ['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{upstream}']
+    )
+    flags = [
+      ...baseGitOptions,
+      upstreamRemote
+    ]
     try {
-      cp.execFileSync('git', [
-        ...baseGitOptions,
-        upstreamRemote
-      ], { stdio: 'pipe' })
+      cachedExec('git', flags)
     } catch (err) {
-      // If the CI is working on a detached HEAD or branch tracking hasn’t been set up, the above command will fail.
-      log.error('Git plugin error executing fallback git command', err)
+      // If the CI is working on a detached HEAD or branch tracking hasn't been set up, the above command will fail.
+      log.warn(`Git unshallow failed again: ${flags.join(' ')}`)
       incrementCountMetric(
         TELEMETRY_GIT_COMMAND_ERRORS,
         { command: 'unshallow', errorType: err.code, exitCode: err.status || err.errno }
@@ -202,7 +209,7 @@ function getLatestCommits () {
   incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'get_local_commits' })
   const startTime = Date.now()
   try {
-    const result = cp.execFileSync('git', ['log', '--format=%H', '-n 1000', '--since="1 month ago"'], { stdio: 'pipe' })
+    const result = cachedExec('git', ['log', '--format=%H', '-n 1000', '--since="1 month ago"'])
       .toString()
       .split('\n')
       .filter(Boolean)
@@ -272,10 +279,9 @@ function checkAndFetchBranch (branch, remoteName) {
   try {
     // `git show-ref --verify --quiet refs/remotes/${remoteName}/${branch}` will exit 0 if the branch exists
     // Otherwise it will exit 1
-    cp.execFileSync(
+    cachedExec(
       'git',
       ['show-ref', '--verify', '--quiet', `refs/remotes/${remoteName}/${branch}`],
-      { stdio: 'pipe' }
     )
     // branch exists locally, so we finish
   } catch {
@@ -285,22 +291,22 @@ function checkAndFetchBranch (branch, remoteName) {
       // `git ls-remote --heads origin my-branch` will exit 0 even if the branch doesn't exist.
       // The piece of information we need is whether the command outputs anything.
       // `git ls-remote --heads origin my-branch` could exit an error code if the remote does not exist.
-      const remoteHeads = cp.execFileSync(
+      const remoteHeads = cachedExec(
         'git',
         ['ls-remote', '--heads', remoteName, branch],
         { stdio: 'pipe', timeout: 2000 }
       )
       if (remoteHeads) {
         // branch exists, so we'll fetch it
-        cp.execFileSync(
+        cachedExec(
           'git',
           ['fetch', '--depth', '1', remoteName, branch],
           { stdio: 'pipe', timeout: 5000 }
         )
       }
-    } catch (e) {
+    } catch (err) {
       // branch does not exist or couldn't be fetched, so we can't do anything
-      log.error('Git plugin error checking and fetching branch', e)
+      log.debug('Git plugin error checking and fetching branch', err)
     }
   }
 }
@@ -358,7 +364,7 @@ function getCommitsRevList (commitsToExclude, commitsToInclude) {
   incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'get_objects' })
   const startTime = Date.now()
   try {
-    result = cp.execFileSync(
+    result = cachedExec(
       'git',
       [
         'rev-list',
@@ -403,7 +409,7 @@ function generatePackFilesForCommits (commitsToUpload) {
   // Generates pack files to upload and
   // returns the ordered list of packfiles' paths
   function execGitPackObjects (targetPath) {
-    return cp.execFileSync(
+    return cachedExec(
       'git',
       [
         'pack-objects',
@@ -450,6 +456,13 @@ function generatePackFilesForCommits (commitsToUpload) {
   return result
 }
 
+function getRepositoryRoot () {
+  return sanitizedExec(
+    'git',
+    ['rev-parse', '--show-toplevel']
+  )
+}
+
 // If there is ciMetadata, it takes precedence.
 function getGitMetadata (ciMetadata) {
   const {
@@ -480,7 +493,7 @@ function getGitMetadata (ciMetadata) {
       commitMessage || sanitizedExec('git', ['show', '-s', '--format=%B'], null, null, null, false),
     [GIT_BRANCH]: branch || sanitizedExec('git', ['rev-parse', '--abbrev-ref', 'HEAD']),
     [GIT_COMMIT_SHA]: commitSHA || sanitizedExec('git', ['rev-parse', 'HEAD']),
-    [CI_WORKSPACE_PATH]: ciWorkspacePath || sanitizedExec('git', ['rev-parse', '--show-toplevel']),
+    [CI_WORKSPACE_PATH]: ciWorkspacePath || getRepositoryRoot(),
   }
 
   if (headCommitSha) {
@@ -596,5 +609,6 @@ module.exports = {
   getLocalBranches,
   getMergeBase,
   getCounts,
-  fetchHeadCommitSha
+  fetchHeadCommitSha,
+  getRepositoryRoot
 }

package/packages/dd-trace/src/plugins/util/test.js

@@ -119,6 +119,10 @@ const MOCHA_WORKER_TRACE_PAYLOAD_CODE = 80
 // playwright worker variables
 const PLAYWRIGHT_WORKER_TRACE_PAYLOAD_CODE = 90
 
+// vitest worker variables
+const VITEST_WORKER_TRACE_PAYLOAD_CODE = 100
+const VITEST_WORKER_LOGS_PAYLOAD_CODE = 102
+
 // Early flake detection util strings
 const EFD_STRING = "Retried by Datadog's Early Flake Detection"
 const EFD_TEST_NAME_REGEX = new RegExp(EFD_STRING + String.raw` \(#\d+\): `, 'g')
@@ -218,6 +222,8 @@ module.exports = {
   CUCUMBER_WORKER_TRACE_PAYLOAD_CODE,
   MOCHA_WORKER_TRACE_PAYLOAD_CODE,
   PLAYWRIGHT_WORKER_TRACE_PAYLOAD_CODE,
+  VITEST_WORKER_TRACE_PAYLOAD_CODE,
+  VITEST_WORKER_LOGS_PAYLOAD_CODE,
   TEST_SOURCE_START,
   TEST_SKIPPED_BY_ITR,
   TEST_IS_NEW,
@@ -438,36 +444,40 @@ function checkShaDiscrepancies (ciMetadata, userProvidedGitMetadata) {
   )
 }
 
-function getTestEnvironmentMetadata (testFramework, config) {
-  // TODO: eventually these will come from the tracer (generally available)
+function getTestEnvironmentMetadata (testFramework, config, shouldSkipGitMetadataExtraction = false) {
   const ciMetadata = getCIMetadata()
-  const {
-    [GIT_COMMIT_SHA]: commitSHA,
-    [GIT_BRANCH]: branch,
-    [GIT_REPOSITORY_URL]: repositoryUrl,
-    [GIT_TAG]: tag,
-    [GIT_COMMIT_AUTHOR_NAME]: authorName,
-    [GIT_COMMIT_AUTHOR_EMAIL]: authorEmail,
-    [GIT_COMMIT_MESSAGE]: commitMessage,
-    [CI_WORKSPACE_PATH]: ciWorkspacePath,
-    [GIT_COMMIT_HEAD_SHA]: headCommitSha
-  } = ciMetadata
-
-  const gitMetadata = getGitMetadata({
-    commitSHA,
-    branch,
-    repositoryUrl,
-    tag,
-    authorName,
-    authorEmail,
-    commitMessage,
-    ciWorkspacePath,
-    headCommitSha
-  })
-
   const userProvidedGitMetadata = getUserProviderGitMetadata()
 
-  checkShaDiscrepancies(ciMetadata, userProvidedGitMetadata)
+  let gitMetadata = {}
+
+  // We don't execute git in test framework workers since the information is in the parent process
+  // and git metadata does not affect the execution of the tests
+  if (!shouldSkipGitMetadataExtraction) {
+    checkShaDiscrepancies(ciMetadata, userProvidedGitMetadata)
+
+    const {
+      [GIT_COMMIT_SHA]: commitSHA,
+      [GIT_BRANCH]: branch,
+      [GIT_REPOSITORY_URL]: repositoryUrl,
+      [GIT_TAG]: tag,
+      [GIT_COMMIT_AUTHOR_NAME]: authorName,
+      [GIT_COMMIT_AUTHOR_EMAIL]: authorEmail,
+      [GIT_COMMIT_MESSAGE]: commitMessage,
+      [CI_WORKSPACE_PATH]: ciWorkspacePath,
+      [GIT_COMMIT_HEAD_SHA]: headCommitSha
+    } = ciMetadata
+    gitMetadata = getGitMetadata({
+      commitSHA,
+      branch,
+      repositoryUrl,
+      tag,
+      authorName,
+      authorEmail,
+      commitMessage,
+      ciWorkspacePath,
+      headCommitSha
+    })
+  }
 
   const runtimeAndOSMetadata = getRuntimeAndOSMetadata()
 

package/packages/dd-trace/src/plugins/util/web.js

@@ -387,7 +387,7 @@ const web = {
 
     return function (statusCode, statusMessage, headers) {
       headers = typeof statusMessage === 'string' ? headers : statusMessage
-      headers = Object.assign(res.getHeaders(), headers)
+      headers = { ...res.getHeaders(), ...headers }
 
       if (req.method.toLowerCase() === 'options' && isOriginAllowed(req, headers)) {
         addAllowHeaders(req, res, headers)

package/packages/dd-trace/src/profiler.js

@@ -10,6 +10,8 @@ module.exports = {
   start: config => {
     const { service, version, env, url, hostname, port, tags, repositoryUrl, commitSHA, injectionEnabled } = config
     const { enabled, sourceMap, exporters } = config.profiling
+    const { heartbeatInterval } = config.telemetry
+
     const logger = {
       debug: (message) => log.debug(message),
       info: (message) => log.info(message),
@@ -39,7 +41,8 @@ module.exports = {
       repositoryUrl,
       commitSHA,
       libraryInjected,
-      activation
+      activation,
+      heartbeatInterval
     })
   },