dd-trace 4.18.0 → 5.6.0

package/packages/dd-trace/src/plugins/ci_plugin.js

@@ -15,11 +15,20 @@ const {
   TEST_MODULE,
   getTestSuiteCommonTags,
   TEST_STATUS,
-  TEST_SKIPPED_BY_ITR
+  TEST_SKIPPED_BY_ITR,
+  ITR_CORRELATION_ID
 } = require('./util/test')
 const Plugin = require('./plugin')
 const { COMPONENT } = require('../constants')
 const log = require('../log')
+const {
+  incrementCountMetric,
+  distributionMetric,
+  TELEMETRY_EVENT_CREATED,
+  TELEMETRY_ITR_SKIPPED
+} = require('../ci-visibility/telemetry')
+const { CI_PROVIDER_NAME, GIT_REPOSITORY_URL, GIT_COMMIT_SHA, GIT_BRANCH, CI_WORKSPACE_PATH } = require('./util/tags')
+const { OS_VERSION, OS_PLATFORM, OS_ARCHITECTURE, RUNTIME_NAME, RUNTIME_VERSION } = require('./util/env')
 
 module.exports = class CiPlugin extends Plugin {
   constructor (...args) {
@@ -27,29 +36,31 @@ module.exports = class CiPlugin extends Plugin {
 
     this.rootDir = process.cwd() // fallback in case :session:start events are not emitted
 
-    this.addSub(`ci:${this.constructor.id}:itr-configuration`, ({ onDone }) => {
-      if (!this.tracer._exporter || !this.tracer._exporter.getItrConfiguration) {
+    this.addSub(`ci:${this.constructor.id}:library-configuration`, ({ onDone }) => {
+      if (!this.tracer._exporter || !this.tracer._exporter.getLibraryConfiguration) {
         return onDone({ err: new Error('CI Visibility was not initialized correctly') })
       }
-      this.tracer._exporter.getItrConfiguration(this.testConfiguration, (err, itrConfig) => {
+      this.tracer._exporter.getLibraryConfiguration(this.testConfiguration, (err, libraryConfig) => {
         if (err) {
           log.error(`Intelligent Test Runner configuration could not be fetched. ${err.message}`)
         } else {
-          this.itrConfig = itrConfig
+          this.libraryConfig = libraryConfig
         }
-        onDone({ err, itrConfig })
+        onDone({ err, libraryConfig })
       })
     })
 
     this.addSub(`ci:${this.constructor.id}:test-suite:skippable`, ({ onDone }) => {
-      if (!this.tracer._exporter || !this.tracer._exporter.getSkippableSuites) {
+      if (!this.tracer._exporter?.getSkippableSuites) {
         return onDone({ err: new Error('CI Visibility was not initialized correctly') })
       }
-      this.tracer._exporter.getSkippableSuites(this.testConfiguration, (err, skippableSuites) => {
+      this.tracer._exporter.getSkippableSuites(this.testConfiguration, (err, skippableSuites, itrCorrelationId) => {
         if (err) {
           log.error(`Skippable suites could not be fetched. ${err.message}`)
+        } else {
+          this.itrCorrelationId = itrCorrelationId
         }
-        onDone({ err, skippableSuites })
+        onDone({ err, skippableSuites, itrCorrelationId })
       })
     })
 
@@ -71,6 +82,7 @@ module.exports = class CiPlugin extends Plugin {
           ...testSessionSpanMetadata
         }
       })
+      this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'session')
       this.testModuleSpan = this.tracer.startSpan(`${this.constructor.id}.test_module`, {
         childOf: this.testSessionSpan,
         tags: {
@@ -79,12 +91,16 @@ module.exports = class CiPlugin extends Plugin {
           ...testModuleSpanMetadata
         }
       })
+      this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'module')
     })
 
     this.addSub(`ci:${this.constructor.id}:itr:skipped-suites`, ({ skippedSuites, frameworkVersion }) => {
       const testCommand = this.testSessionSpan.context()._tags[TEST_COMMAND]
       skippedSuites.forEach((testSuite) => {
         const testSuiteMetadata = getTestSuiteCommonTags(testCommand, frameworkVersion, testSuite, this.constructor.id)
+        if (this.itrCorrelationId) {
+          testSuiteMetadata[ITR_CORRELATION_ID] = this.itrCorrelationId
+        }
 
         this.tracer.startSpan(`${this.constructor.id}.test_suite`, {
           childOf: this.testModuleSpan,
@@ -97,25 +113,65 @@ module.exports = class CiPlugin extends Plugin {
           }
         }).finish()
       })
+      this.telemetry.count(TELEMETRY_ITR_SKIPPED, { testLevel: 'suite' }, skippedSuites.length)
+    })
+
+    this.addSub(`ci:${this.constructor.id}:known-tests`, ({ onDone }) => {
+      if (!this.tracer._exporter?.getKnownTests) {
+        return onDone({ err: new Error('CI Visibility was not initialized correctly') })
+      }
+      this.tracer._exporter.getKnownTests(this.testConfiguration, (err, knownTests) => {
+        if (err) {
+          log.error(`Known tests could not be fetched. ${err.message}`)
+        }
+        onDone({ err, knownTests })
+      })
     })
   }
 
+  get telemetry () {
+    const testFramework = this.constructor.id
+    return {
+      ciVisEvent: function (name, testLevel, tags = {}) {
+        incrementCountMetric(name, {
+          testLevel,
+          testFramework,
+          isUnsupportedCIProvider: this.isUnsupportedCIProvider,
+          ...tags
+        })
+      },
+      count: function (name, tags, value = 1) {
+        incrementCountMetric(name, tags, value)
+      },
+      distribution: function (name, tags, measure) {
+        distributionMetric(name, tags, measure)
+      }
+    }
+  }
+
   configure (config) {
     super.configure(config)
     this.testEnvironmentMetadata = getTestEnvironmentMetadata(this.constructor.id, this.config)
-    this.codeOwnersEntries = getCodeOwnersFileEntries()
 
     const {
-      'git.repository_url': repositoryUrl,
-      'git.commit.sha': sha,
-      'os.version': osVersion,
-      'os.platform': osPlatform,
-      'os.architecture': osArchitecture,
-      'runtime.name': runtimeName,
-      'runtime.version': runtimeVersion,
-      'git.branch': branch
+      [GIT_REPOSITORY_URL]: repositoryUrl,
+      [GIT_COMMIT_SHA]: sha,
+      [OS_VERSION]: osVersion,
+      [OS_PLATFORM]: osPlatform,
+      [OS_ARCHITECTURE]: osArchitecture,
+      [RUNTIME_NAME]: runtimeName,
+      [RUNTIME_VERSION]: runtimeVersion,
+      [GIT_BRANCH]: branch,
+      [CI_PROVIDER_NAME]: ciProviderName,
+      [CI_WORKSPACE_PATH]: repositoryRoot
     } = this.testEnvironmentMetadata
 
+    this.repositoryRoot = repositoryRoot || process.cwd()
+
+    this.codeOwnersEntries = getCodeOwnersFileEntries(repositoryRoot)
+
+    this.isUnsupportedCIProvider = !ciProviderName
+
     this.testConfiguration = {
       repositoryUrl,
       sha,
@@ -124,7 +180,8 @@ module.exports = class CiPlugin extends Plugin {
       osArchitecture,
       runtimeName,
       runtimeVersion,
-      branch
+      branch,
+      testLevel: 'suite'
     }
   }
 
@@ -169,6 +226,8 @@ module.exports = class CiPlugin extends Plugin {
       }
     }
 
+    this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'test', { hasCodeOwners: !!codeOwners })
+
     const testSpan = this.tracer
       .startSpan(`${this.constructor.id}.test`, {
         childOf,

package/packages/dd-trace/src/plugins/database.js

@@ -36,7 +36,7 @@ class DatabasePlugin extends StoragePlugin {
     const { encodedDddbs, encodedDde, encodedDdps, encodedDdpv } = this.serviceTags
 
     return `dddbs='${encodedDddbs}',dde='${encodedDde}',` +
-    `ddps='${encodedDdps}',ddpv='${encodedDdpv}'`
+      `ddps='${encodedDdps}',ddpv='${encodedDdpv}'`
   }
 
   getDbmServiceName (span, tracerService) {

package/packages/dd-trace/src/plugins/index.js

@@ -17,11 +17,13 @@ module.exports = {
   get '@opensearch-project/opensearch' () { return require('../../../datadog-plugin-opensearch/src') },
   get '@redis/client' () { return require('../../../datadog-plugin-redis/src') },
   get '@smithy/smithy-client' () { return require('../../../datadog-plugin-aws-sdk/src') },
+  get 'aerospike' () { return require('../../../datadog-plugin-aerospike/src') },
   get 'amqp10' () { return require('../../../datadog-plugin-amqp10/src') },
   get 'amqplib' () { return require('../../../datadog-plugin-amqplib/src') },
   get 'aws-sdk' () { return require('../../../datadog-plugin-aws-sdk/src') },
   get 'bunyan' () { return require('../../../datadog-plugin-bunyan/src') },
   get 'cassandra-driver' () { return require('../../../datadog-plugin-cassandra-driver/src') },
+  get 'child_process' () { return require('../../../datadog-plugin-child_process/src') },
   get 'connect' () { return require('../../../datadog-plugin-connect/src') },
   get 'couchbase' () { return require('../../../datadog-plugin-couchbase/src') },
   get 'cypress' () { return require('../../../datadog-plugin-cypress/src') },
@@ -58,6 +60,11 @@ module.exports = {
   get 'mysql2' () { return require('../../../datadog-plugin-mysql2/src') },
   get 'net' () { return require('../../../datadog-plugin-net/src') },
   get 'next' () { return require('../../../datadog-plugin-next/src') },
+  get 'node:dns' () { return require('../../../datadog-plugin-dns/src') },
+  get 'node:http' () { return require('../../../datadog-plugin-http/src') },
+  get 'node:http2' () { return require('../../../datadog-plugin-http2/src') },
+  get 'node:https' () { return require('../../../datadog-plugin-http/src') },
+  get 'node:net' () { return require('../../../datadog-plugin-net/src') },
   get 'oracledb' () { return require('../../../datadog-plugin-oracledb/src') },
   get 'openai' () { return require('../../../datadog-plugin-openai/src') },
   get 'paperplane' () { return require('../../../datadog-plugin-paperplane/src') },

package/packages/dd-trace/src/plugins/plugin.js

@@ -2,7 +2,7 @@
 
 // TODO: move anything related to tracing to TracingPlugin instead
 
-const dc = require('../../../diagnostics_channel')
+const dc = require('dc-polyfill')
 const { storage } = require('../../../datadog-core')
 
 class Subscription {

package/packages/dd-trace/src/plugins/util/ci.js

@@ -1,5 +1,3 @@
-const URL = require('url').URL
-
 const {
   GIT_BRANCH,
   GIT_COMMIT_SHA,
@@ -24,6 +22,7 @@ const {
   CI_NODE_LABELS,
   CI_NODE_NAME
 } = require('./tags')
+const { filterSensitiveInfoFromRepository } = require('./url')
 
 // Receives a string with the form 'John Doe <john.doe@gmail.com>'
 // and returns { name: 'John Doe', email: 'john.doe@gmail.com' }
@@ -67,20 +66,6 @@ function normalizeRef (ref) {
   return ref.replace(/origin\/|refs\/heads\/|tags\//gm, '')
 }
 
-function filterSensitiveInfoFromRepository (repositoryUrl) {
-  if (repositoryUrl.startsWith('git@')) {
-    return repositoryUrl
-  }
-
-  try {
-    const { protocol, hostname, pathname } = new URL(repositoryUrl)
-
-    return `${protocol}//${hostname}${pathname}`
-  } catch (e) {
-    return ''
-  }
-}
-
 function resolveTilde (filePath) {
   if (!filePath || typeof filePath !== 'string') {
     return ''
@@ -271,20 +256,22 @@ module.exports = {
       const ref = GITHUB_HEAD_REF || GITHUB_REF || ''
       const refKey = ref.includes('tags/') ? GIT_TAG : GIT_BRANCH
 
+      // Both pipeline URL and job URL include GITHUB_SERVER_URL, which can include user credentials,
+      // so we pass them through `filterSensitiveInfoFromRepository`.
       tags = {
         [CI_PIPELINE_ID]: GITHUB_RUN_ID,
         [CI_PIPELINE_NAME]: GITHUB_WORKFLOW,
         [CI_PIPELINE_NUMBER]: GITHUB_RUN_NUMBER,
-        [CI_PIPELINE_URL]: pipelineURL,
+        [CI_PIPELINE_URL]: filterSensitiveInfoFromRepository(pipelineURL),
         [CI_PROVIDER_NAME]: 'github',
         [GIT_COMMIT_SHA]: GITHUB_SHA,
         [GIT_REPOSITORY_URL]: repositoryURL,
-        [CI_JOB_URL]: jobUrl,
+        [CI_JOB_URL]: filterSensitiveInfoFromRepository(jobUrl),
         [CI_JOB_NAME]: GITHUB_JOB,
         [CI_WORKSPACE_PATH]: GITHUB_WORKSPACE,
         [refKey]: ref,
         [CI_ENV_VARS]: JSON.stringify({
-          GITHUB_SERVER_URL,
+          GITHUB_SERVER_URL: filterSensitiveInfoFromRepository(GITHUB_SERVER_URL),
           GITHUB_REPOSITORY,
           GITHUB_RUN_ID,
           GITHUB_RUN_ATTEMPT

package/packages/dd-trace/src/plugins/util/git.js

@@ -1,10 +1,9 @@
-const { execFileSync } = require('child_process')
+const cp = require('child_process')
 const os = require('os')
 const path = require('path')
 const fs = require('fs')
 
 const log = require('../../log')
-const { sanitizedExec } = require('./exec')
 const {
   GIT_COMMIT_SHA,
   GIT_BRANCH,
@@ -19,9 +18,52 @@ const {
   GIT_COMMIT_AUTHOR_NAME,
   CI_WORKSPACE_PATH
 } = require('./tags')
+const {
+  incrementCountMetric,
+  distributionMetric,
+  TELEMETRY_GIT_COMMAND,
+  TELEMETRY_GIT_COMMAND_MS,
+  TELEMETRY_GIT_COMMAND_ERRORS
+} = require('../../ci-visibility/telemetry')
+const { filterSensitiveInfoFromRepository } = require('./url')
+const { storage } = require('../../../../datadog-core')
 
 const GIT_REV_LIST_MAX_BUFFER = 8 * 1024 * 1024 // 8MB
 
+function sanitizedExec (
+  cmd,
+  flags,
+  operationMetric,
+  durationMetric,
+  errorMetric
+) {
+  const store = storage.getStore()
+  storage.enterWith({ noop: true })
+
+  let startTime
+  if (operationMetric) {
+    incrementCountMetric(operationMetric.name, operationMetric.tags)
+  }
+  if (durationMetric) {
+    startTime = Date.now()
+  }
+  try {
+    const result = cp.execFileSync(cmd, flags, { stdio: 'pipe' }).toString().replace(/(\r\n|\n|\r)/gm, '')
+    if (durationMetric) {
+      distributionMetric(durationMetric.name, durationMetric.tags, Date.now() - startTime)
+    }
+    return result
+  } catch (e) {
+    if (errorMetric) {
+      incrementCountMetric(errorMetric.name, { ...errorMetric.tags, exitCode: e.status })
+    }
+    log.error(e)
+    return ''
+  } finally {
+    storage.enterWith(store)
+  }
+}
+
 function isDirectory (path) {
   try {
     const stats = fs.statSync(path)
@@ -32,7 +74,13 @@ function isDirectory (path) {
 }
 
 function isShallowRepository () {
-  return sanitizedExec('git', ['rev-parse', '--is-shallow-repository']) === 'true'
+  return sanitizedExec(
+    'git',
+    ['rev-parse', '--is-shallow-repository'],
+    { name: TELEMETRY_GIT_COMMAND, tags: { command: 'check_shallow' } },
+    { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'check_shallow' } },
+    { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'check_shallow' } }
+  ) === 'true'
 }
 
 function getGitVersion () {
@@ -71,50 +119,76 @@ function unshallowRepository () {
     defaultRemoteName
   ]
 
+  incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'unshallow' })
+  const start = Date.now()
   try {
-    execFileSync('git', [
+    cp.execFileSync('git', [
       ...baseGitOptions,
       revParseHead
     ], { stdio: 'pipe' })
-  } catch (e) {
+  } catch (err) {
     // If the local HEAD is a commit that has not been pushed to the remote, the above command will fail.
-    log.error(e)
+    log.error(err)
+    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'unshallow', exitCode: err.status })
     const upstreamRemote = sanitizedExec('git', ['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{upstream}'])
     try {
-      execFileSync('git', [
+      cp.execFileSync('git', [
         ...baseGitOptions,
         upstreamRemote
       ], { stdio: 'pipe' })
-    } catch (e) {
+    } catch (err) {
       // If the CI is working on a detached HEAD or branch tracking hasn’t been set up, the above command will fail.
-      log.error(e)
+      log.error(err)
+      incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'unshallow', exitCode: err.status })
       // We use sanitizedExec here because if this last option fails, we'll give up.
-      sanitizedExec('git', baseGitOptions)
+      sanitizedExec(
+        'git',
+        baseGitOptions,
+        null,
+        null,
+        { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'unshallow' } } // we log the error in sanitizedExec
+      )
     }
   }
+  distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'unshallow' }, Date.now() - start)
 }
 
 function getRepositoryUrl () {
-  return sanitizedExec('git', ['config', '--get', 'remote.origin.url'])
+  return sanitizedExec(
+    'git',
+    ['config', '--get', 'remote.origin.url'],
+    { name: TELEMETRY_GIT_COMMAND, tags: { command: 'get_repository' } },
+    { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'get_repository' } },
+    { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'get_repository' } }
+  )
 }
 
 function getLatestCommits () {
+  incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'get_local_commits' })
+  const startTime = Date.now()
   try {
-    return execFileSync('git', ['log', '--format=%H', '-n 1000', '--since="1 month ago"'], { stdio: 'pipe' })
+    const result = cp.execFileSync('git', ['log', '--format=%H', '-n 1000', '--since="1 month ago"'], { stdio: 'pipe' })
       .toString()
       .split('\n')
       .filter(commit => commit)
+    distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'get_local_commits' }, Date.now() - startTime)
+    return result
   } catch (err) {
     log.error(`Get latest commits failed: ${err.message}`)
+    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'get_local_commits', errorType: err.status })
     return []
   }
 }
 
-function getCommitsToUpload (commitsToExclude, commitsToInclude) {
+function getCommitsRevList (commitsToExclude, commitsToInclude) {
+  let result = []
+
   const commitsToExcludeString = commitsToExclude.map(commit => `^${commit}`)
 
+  incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'get_objects' })
+  const startTime = Date.now()
   try {
-    return execFileSync(
+    result = cp.execFileSync(
       'git',
       [
         'rev-list',
@@ -131,11 +205,14 @@ function getCommitsToUpload (commitsToExclude, commitsToInclude) {
       .filter(commit => commit)
   } catch (err) {
     log.error(`Get commits to upload failed: ${err.message}`)
-    return []
+    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'get_objects', errorType: err.status })
   }
+  distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'get_objects' }, Date.now() - startTime)
+  return result
 }
 
 function generatePackFilesForCommits (commitsToUpload) {
+  let result = []
   const tmpFolder = os.tmpdir()
 
   if (!isDirectory(tmpFolder)) {
@@ -147,10 +224,12 @@ function generatePackFilesForCommits (commitsToUpload) {
   const temporaryPath = path.join(tmpFolder, randomPrefix)
   const cwdPath = path.join(process.cwd(), randomPrefix)
 
+  incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'pack_objects' })
+  const startTime = Date.now()
   // Generates pack files to upload and
   // returns the ordered list of packfiles' paths
   function execGitPackObjects (targetPath) {
-    return execFileSync(
+    return cp.execFileSync(
       'git',
       [
         'pack-objects',
@@ -163,9 +242,10 @@ function generatePackFilesForCommits (commitsToUpload) {
   }
 
   try {
-    return execGitPackObjects(temporaryPath)
+    result = execGitPackObjects(temporaryPath)
   } catch (err) {
     log.error(err)
+    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'pack_objects', errorType: err.status })
     /**
      * The generation of pack files in the temporary folder (from `os.tmpdir()`)
      * sometimes fails in certain CI setups with the error message
@@ -179,13 +259,15 @@ function generatePackFilesForCommits (commitsToUpload) {
      * TODO: fix issue and remove workaround.
      */
     try {
-      return execGitPackObjects(cwdPath)
+      result = execGitPackObjects(cwdPath)
     } catch (err) {
       log.error(err)
+      incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'pack_objects', errorType: err.status })
     }
-
-    return []
   }
+  distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'pack_objects' }, Date.now() - startTime)
+
+  return result
 }
 
 // If there is ciMetadata, it takes precedence.
@@ -214,7 +296,7 @@ function getGitMetadata (ciMetadata) {
 
   return {
     [GIT_REPOSITORY_URL]:
-      repositoryUrl || sanitizedExec('git', ['ls-remote', '--get-url']),
+      filterSensitiveInfoFromRepository(repositoryUrl || sanitizedExec('git', ['ls-remote', '--get-url'])),
     [GIT_COMMIT_MESSAGE]:
       commitMessage || sanitizedExec('git', ['show', '-s', '--format=%s']),
     [GIT_COMMIT_AUTHOR_DATE]: authorDate,
@@ -235,7 +317,7 @@ module.exports = {
   getLatestCommits,
   getRepositoryUrl,
   generatePackFilesForCommits,
-  getCommitsToUpload,
+  getCommitsRevList,
   GIT_REV_LIST_MAX_BUFFER,
   isShallowRepository,
   unshallowRepository

package/packages/dd-trace/src/plugins/util/ip_extractor.js

@@ -48,8 +48,8 @@ function extractIp (config, req) {
 
   let firstPrivateIp
   if (headers) {
-    for (let i = 0; i < ipHeaderList.length; i++) {
-      const firstIp = findFirstIp(headers[ipHeaderList[i]])
+    for (const ipHeaderName of ipHeaderList) {
+      const firstIp = findFirstIp(headers[ipHeaderName])
 
       if (firstIp.public) {
         return firstIp.public
@@ -59,7 +59,7 @@ function extractIp (config, req) {
     }
   }
 
-  return firstPrivateIp || (req.socket && req.socket.remoteAddress)
+  return firstPrivateIp || req.socket?.remoteAddress
 }
 
 function findFirstIp (str) {
@@ -68,8 +68,8 @@ function findFirstIp (str) {
 
   const splitted = str.split(',')
 
-  for (let i = 0; i < splitted.length; i++) {
-    const chunk = splitted[i].trim()
+  for (const part of splitted) {
+    const chunk = part.trim()
 
     // TODO: strip port and interface data ?
 
@@ -90,5 +90,6 @@ function findFirstIp (str) {
 }
 
 module.exports = {
-  extractIp
+  extractIp,
+  ipHeaderList
 }