dd-trace 4.18.0 → 5.6.0

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -5,6 +5,16 @@ const log = require('../../../log')
 
 const { AgentlessCiVisibilityEncoder } = require('../../../encode/agentless-ci-visibility')
 const BaseWriter = require('../../../exporters/common/writer')
+const {
+  incrementCountMetric,
+  distributionMetric,
+  TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS,
+  TELEMETRY_ENDPOINT_PAYLOAD_BYTES,
+  TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS,
+  TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
+  TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
+  getErrorTypeFromStatusCode
+} = require('../../../ci-visibility/telemetry')
 
 class Writer extends BaseWriter {
   constructor ({ url, tags, evpProxyPrefix = '' }) {
@@ -35,8 +45,27 @@ class Writer extends BaseWriter {
 
     log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
 
-    request(data, options, (err, res) => {
+    const startRequestTime = Date.now()
+
+    incrementCountMetric(TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, { endpoint: 'test_cycle' })
+    distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_BYTES, { endpoint: 'test_cycle' }, data.length)
+
+    request(data, options, (err, res, statusCode) => {
+      distributionMetric(
+        TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS,
+        { endpoint: 'test_cycle' },
+        Date.now() - startRequestTime
+      )
       if (err) {
+        const errorType = getErrorTypeFromStatusCode(statusCode)
+        incrementCountMetric(
+          TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
+          { endpoint: 'test_cycle', errorType }
+        )
+        incrementCountMetric(
+          TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
+          { endpoint: 'test_cycle' }
+        )
         log.error(err)
         done()
         return

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -3,8 +3,9 @@
 const URL = require('url').URL
 
 const { sendGitMetadata: sendGitMetadataRequest } = require('./git/git_metadata')
-const { getItrConfiguration: getItrConfigurationRequest } = require('../intelligent-test-runner/get-itr-configuration')
+const { getLibraryConfiguration: getLibraryConfigurationRequest } = require('../requests/get-library-configuration')
 const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligent-test-runner/get-skippable-suites')
+const { getKnownTests: getKnownTestsRequest } = require('../early-flake-detection/get-known-tests')
 const log = require('../../log')
 const AgentInfoExporter = require('../../exporters/common/agent-info-exporter')
 
@@ -76,11 +77,18 @@ class CiVisibilityExporter extends AgentInfoExporter {
   shouldRequestSkippableSuites () {
     return !!(this._config.isIntelligentTestRunnerEnabled &&
       this._canUseCiVisProtocol &&
-      this._itrConfig &&
-      this._itrConfig.isSuitesSkippingEnabled)
+      this._libraryConfig?.isSuitesSkippingEnabled)
   }
 
-  shouldRequestItrConfiguration () {
+  shouldRequestKnownTests () {
+    return !!(
+      this._config.isEarlyFlakeDetectionEnabled &&
+      this._canUseCiVisProtocol &&
+      this._libraryConfig?.isEarlyFlakeDetectionEnabled
+    )
+  }
+
+  shouldRequestLibraryConfiguration () {
     return this._config.isIntelligentTestRunnerEnabled
   }
 
@@ -92,6 +100,19 @@ class CiVisibilityExporter extends AgentInfoExporter {
     return this._canUseCiVisProtocol
   }
 
+  getRequestConfiguration (testConfiguration) {
+    return {
+      url: this._getApiUrl(),
+      env: this._config.env,
+      service: this._config.service,
+      isEvpProxy: !!this._isUsingEvpProxy,
+      isGzipCompatible: this._isGzipCompatible,
+      evpProxyPrefix: this.evpProxyPrefix,
+      custom: getTestConfigurationTags(this._config.tags),
+      ...testConfiguration
+    }
+  }
+
   // We can't call the skippable endpoint until git upload has finished,
   // hence the this._gitUploadPromise.then
   getSkippableSuites (testConfiguration, callback) {
@@ -102,52 +123,84 @@ class CiVisibilityExporter extends AgentInfoExporter {
       if (gitUploadError) {
         return callback(gitUploadError, [])
       }
-      const configuration = {
-        url: this._getApiUrl(),
-        site: this._config.site,
-        env: this._config.env,
-        service: this._config.service,
-        isEvpProxy: !!this._isUsingEvpProxy,
-        custom: getTestConfigurationTags(this._config.tags),
-        ...testConfiguration
-      }
-      getSkippableSuitesRequest(configuration, callback)
+      getSkippableSuitesRequest(this.getRequestConfiguration(testConfiguration), callback)
     })
   }
 
+  getKnownTests (testConfiguration, callback) {
+    if (!this.shouldRequestKnownTests()) {
+      return callback(null)
+    }
+    getKnownTestsRequest(this.getRequestConfiguration(testConfiguration), callback)
+  }
+
   /**
-   * We can't request ITR configuration until we know whether we can use the
+   * We can't request library configuration until we know whether we can use the
    * CI Visibility Protocol, hence the this._canUseCiVisProtocol promise.
    */
-  getItrConfiguration (testConfiguration, callback) {
+  getLibraryConfiguration (testConfiguration, callback) {
     const { repositoryUrl } = testConfiguration
     this.sendGitMetadata(repositoryUrl)
-    if (!this.shouldRequestItrConfiguration()) {
+    if (!this.shouldRequestLibraryConfiguration()) {
       return callback(null, {})
     }
     this._canUseCiVisProtocolPromise.then((canUseCiVisProtocol) => {
       if (!canUseCiVisProtocol) {
         return callback(null, {})
       }
-      const configuration = {
-        url: this._getApiUrl(),
-        env: this._config.env,
-        service: this._config.service,
-        isEvpProxy: !!this._isUsingEvpProxy,
-        custom: getTestConfigurationTags(this._config.tags),
-        ...testConfiguration
-      }
-      getItrConfigurationRequest(configuration, (err, itrConfig) => {
+      const configuration = this.getRequestConfiguration(testConfiguration)
+
+      getLibraryConfigurationRequest(configuration, (err, libraryConfig) => {
         /**
-         * **Important**: this._itrConfig remains empty in testing frameworks
-         * where the tests run in a subprocess, because `getItrConfiguration` is called only once.
+         * **Important**: this._libraryConfig remains empty in testing frameworks
+         * where the tests run in a subprocess, like Jest,
+         * because `getLibraryConfiguration` is called only once in the main process.
          */
-        this._itrConfig = itrConfig
-        callback(err, itrConfig)
+        this._libraryConfig = this.filterConfiguration(libraryConfig)
+
+        if (err) {
+          callback(err, {})
+        } else if (libraryConfig?.requireGit) {
+          // If the backend requires git, we'll wait for the upload to finish and request settings again
+          this._gitUploadPromise.then(gitUploadError => {
+            if (gitUploadError) {
+              return callback(gitUploadError, {})
+            }
+            getLibraryConfigurationRequest(configuration, (err, finalLibraryConfig) => {
+              this._libraryConfig = this.filterConfiguration(finalLibraryConfig)
+              callback(err, this._libraryConfig)
+            })
+          })
+        } else {
+          callback(null, this._libraryConfig)
+        }
       })
     })
   }
 
+  // Takes into account potential kill switches
+  filterConfiguration (remoteConfiguration) {
+    if (!remoteConfiguration) {
+      return {}
+    }
+    const {
+      isCodeCoverageEnabled,
+      isSuitesSkippingEnabled,
+      isItrEnabled,
+      requireGit,
+      isEarlyFlakeDetectionEnabled,
+      earlyFlakeDetectionNumRetries
+    } = remoteConfiguration
+    return {
+      isCodeCoverageEnabled,
+      isSuitesSkippingEnabled,
+      isItrEnabled,
+      requireGit,
+      isEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled && this._config.isEarlyFlakeDetectionEnabled,
+      earlyFlakeDetectionNumRetries
+    }
+  }
+
   sendGitMetadata (repositoryUrl) {
     if (!this._config.isGitUploadEnabled) {
       return
@@ -156,14 +209,19 @@ class CiVisibilityExporter extends AgentInfoExporter {
       if (!canUseCiVisProtocol) {
         return
       }
-      sendGitMetadataRequest(this._getApiUrl(), !!this._isUsingEvpProxy, repositoryUrl, (err) => {
-        if (err) {
-          log.error(`Error uploading git metadata: ${err.message}`)
-        } else {
-          log.debug('Successfully uploaded git metadata')
+      sendGitMetadataRequest(
+        this._getApiUrl(),
+        { isEvpProxy: !!this._isUsingEvpProxy, evpProxyPrefix: this.evpProxyPrefix },
+        repositoryUrl,
+        (err) => {
+          if (err) {
+            log.error(`Error uploading git metadata: ${err.message}`)
+          } else {
+            log.debug('Successfully uploaded git metadata')
+          }
+          this._resolveGit(err)
         }
-        this._resolveGit(err)
-      })
+      )
     })
   }
 

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -10,11 +10,25 @@ const {
   getLatestCommits,
   getRepositoryUrl,
   generatePackFilesForCommits,
-  getCommitsToUpload,
+  getCommitsRevList,
   isShallowRepository,
   unshallowRepository
 } = require('../../../plugins/util/git')
 
+const {
+  incrementCountMetric,
+  distributionMetric,
+  TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS,
+  TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_MS,
+  TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS,
+  TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_NUM,
+  TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES,
+  TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS,
+  TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS,
+  TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES,
+  getErrorTypeFromStatusCode
+} = require('../../../ci-visibility/telemetry')
+
 const isValidSha1 = (sha) => /^[0-9a-f]{40}$/.test(sha)
 const isValidSha256 = (sha) => /^[0-9a-f]{64}$/.test(sha)
 
@@ -46,11 +60,7 @@ function getCommonRequestOptions (url) {
  * The response are the commits for which the backend already has information
  * This response is used to know which commits can be ignored from there on
  */
-function getCommitsToExclude ({ url, isEvpProxy, repositoryUrl }, callback) {
-  const latestCommits = getLatestCommits()
-
-  log.debug(`There were ${latestCommits.length} commits since last month.`)
-
+function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy, evpProxyPrefix }, callback) {
   const commonOptions = getCommonRequestOptions(url)
 
   const options = {
@@ -63,7 +73,7 @@ function getCommitsToExclude ({ url, isEvpProxy, repositoryUrl }, callback) {
   }
 
   if (isEvpProxy) {
-    options.path = '/evp_proxy/v2/api/v2/git/repository/search_commits'
+    options.path = `${evpProxyPrefix}/api/v2/git/repository/search_commits`
     options.headers['X-Datadog-EVP-Subdomain'] = 'api'
     delete options.headers['dd-api-key']
   }
@@ -78,25 +88,41 @@ function getCommitsToExclude ({ url, isEvpProxy, repositoryUrl }, callback) {
     }))
   })
 
-  request(localCommitData, options, (err, response) => {
+  incrementCountMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS)
+  const startTime = Date.now()
+  request(localCommitData, options, (err, response, statusCode) => {
+    distributionMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_MS, {}, Date.now() - startTime)
     if (err) {
+      const errorType = getErrorTypeFromStatusCode(statusCode)
+      incrementCountMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, { errorType })
       const error = new Error(`Error fetching commits to exclude: ${err.message}`)
       return callback(error)
     }
-    let commitsToExclude
+    let alreadySeenCommits
     try {
-      commitsToExclude = validateCommits(JSON.parse(response).data)
+      alreadySeenCommits = validateCommits(JSON.parse(response).data)
     } catch (e) {
+      incrementCountMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, { errorType: 'network' })
       return callback(new Error(`Can't parse commits to exclude response: ${e.message}`))
     }
-    callback(null, commitsToExclude, latestCommits)
+    log.debug(`There are ${alreadySeenCommits.length} commits to exclude.`)
+    const commitsToInclude = latestCommits.filter((commit) => !alreadySeenCommits.includes(commit))
+    log.debug(`There are ${commitsToInclude.length} commits to include.`)
+
+    if (!commitsToInclude.length) {
+      return callback(null, [])
+    }
+
+    const commitsToUpload = getCommitsRevList(alreadySeenCommits, commitsToInclude)
+
+    callback(null, commitsToUpload)
   })
 }
 
 /**
  * This function uploads a git packfile
  */
-function uploadPackFile ({ url, isEvpProxy, packFileToUpload, repositoryUrl, headCommit }, callback) {
+function uploadPackFile ({ url, isEvpProxy, evpProxyPrefix, packFileToUpload, repositoryUrl, headCommit }, callback) {
   const form = new FormData()
 
   const pushedSha = JSON.stringify({
@@ -136,24 +162,86 @@ function uploadPackFile ({ url, isEvpProxy, packFileToUpload, repositoryUrl, hea
   }
 
   if (isEvpProxy) {
-    options.path = '/evp_proxy/v2/api/v2/git/repository/packfile'
+    options.path = `${evpProxyPrefix}/api/v2/git/repository/packfile`
     options.headers['X-Datadog-EVP-Subdomain'] = 'api'
     delete options.headers['dd-api-key']
   }
 
+  incrementCountMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES)
+
+  const uploadSize = form.size()
+
+  const startTime = Date.now()
   request(form, options, (err, _, statusCode) => {
+    distributionMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS, {}, Date.now() - startTime)
     if (err) {
+      const errorType = getErrorTypeFromStatusCode(statusCode)
+      incrementCountMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS, { errorType })
       const error = new Error(`Could not upload packfiles: status code ${statusCode}: ${err.message}`)
-      return callback(error)
+      return callback(error, uploadSize)
     }
-    callback(null)
+    callback(null, uploadSize)
   })
 }
 
+function generateAndUploadPackFiles ({
+  url,
+  isEvpProxy,
+  evpProxyPrefix,
+  commitsToUpload,
+  repositoryUrl,
+  headCommit
+}, callback) {
+  log.debug(`There are ${commitsToUpload.length} commits to upload`)
+
+  const packFilesToUpload = generatePackFilesForCommits(commitsToUpload)
+
+  log.debug(`Uploading ${packFilesToUpload.length} packfiles.`)
+
+  if (!packFilesToUpload.length) {
+    return callback(new Error('Failed to generate packfiles'))
+  }
+
+  distributionMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_NUM, {}, packFilesToUpload.length)
+  let packFileIndex = 0
+  let totalUploadedBytes = 0
+  // This uploads packfiles sequentially
+  const uploadPackFileCallback = (err, byteLength) => {
+    totalUploadedBytes += byteLength
+    if (err || packFileIndex === packFilesToUpload.length) {
+      distributionMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES, {}, totalUploadedBytes)
+      return callback(err)
+    }
+    return uploadPackFile(
+      {
+        packFileToUpload: packFilesToUpload[packFileIndex++],
+        url,
+        isEvpProxy,
+        evpProxyPrefix,
+        repositoryUrl,
+        headCommit
+      },
+      uploadPackFileCallback
+    )
+  }
+
+  uploadPackFile(
+    {
+      packFileToUpload: packFilesToUpload[packFileIndex++],
+      url,
+      isEvpProxy,
+      evpProxyPrefix,
+      repositoryUrl,
+      headCommit
+    },
+    uploadPackFileCallback
+  )
+}
+
 /**
  * This function uploads git metadata to CI Visibility's backend.
 */
-function sendGitMetadata (url, isEvpProxy, configRepositoryUrl, callback) {
+function sendGitMetadata (url, { isEvpProxy, evpProxyPrefix }, configRepositoryUrl, callback) {
   let repositoryUrl = configRepositoryUrl
   if (!repositoryUrl) {
     repositoryUrl = getRepositoryUrl()
@@ -165,65 +253,50 @@ function sendGitMetadata (url, isEvpProxy, configRepositoryUrl, callback) {
     return callback(new Error('Repository URL is empty'))
   }
 
-  if (isShallowRepository()) {
-    log.debug('It is shallow clone, unshallowing...')
-    unshallowRepository()
-  }
+  const latestCommits = getLatestCommits()
+  log.debug(`There were ${latestCommits.length} commits since last month.`)
+  const [headCommit] = latestCommits
 
-  getCommitsToExclude({ url, repositoryUrl, isEvpProxy }, (err, commitsToExclude, latestCommits) => {
+  const getOnFinishGetCommitsToUpload = (hasCheckedShallow) => (err, commitsToUpload) => {
     if (err) {
       return callback(err)
     }
-    log.debug(`There are ${commitsToExclude.length} commits to exclude.`)
-    const [headCommit] = latestCommits
-    const commitsToInclude = latestCommits.filter((commit) => !commitsToExclude.includes(commit))
-    log.debug(`There are ${commitsToInclude.length} commits to include.`)
-
-    const commitsToUpload = getCommitsToUpload(commitsToExclude, commitsToInclude)
 
     if (!commitsToUpload.length) {
       log.debug('No commits to upload')
       return callback(null)
     }
-    log.debug(`There are ${commitsToUpload.length} commits to upload`)
 
-    const packFilesToUpload = generatePackFilesForCommits(commitsToUpload)
-
-    log.debug(`Uploading ${packFilesToUpload.length} packfiles.`)
-
-    if (!packFilesToUpload.length) {
-      return callback(new Error('Failed to generate packfiles'))
-    }
-
-    let packFileIndex = 0
-    // This uploads packfiles sequentially
-    const uploadPackFileCallback = (err) => {
-      if (err || packFileIndex === packFilesToUpload.length) {
-        return callback(err)
-      }
-      return uploadPackFile(
-        {
-          packFileToUpload: packFilesToUpload[packFileIndex++],
-          url,
-          isEvpProxy,
-          repositoryUrl,
-          headCommit
-        },
-        uploadPackFileCallback
-      )
-    }
-
-    uploadPackFile(
-      {
-        packFileToUpload: packFilesToUpload[packFileIndex++],
+    // If it has already unshallowed or the clone is not shallow, we move on
+    if (hasCheckedShallow || !isShallowRepository()) {
+      return generateAndUploadPackFiles({
         url,
         isEvpProxy,
+        evpProxyPrefix,
+        commitsToUpload,
         repositoryUrl,
         headCommit
-      },
-      uploadPackFileCallback
-    )
-  })
+      }, callback)
+    }
+    // Otherwise we unshallow and get commits to upload again
+    log.debug('It is shallow clone, unshallowing...')
+    unshallowRepository()
+    getCommitsToUpload({
+      url,
+      repositoryUrl,
+      latestCommits,
+      isEvpProxy,
+      evpProxyPrefix
+    }, getOnFinishGetCommitsToUpload(true))
+  }
+
+  getCommitsToUpload({
+    url,
+    repositoryUrl,
+    latestCommits,
+    isEvpProxy,
+    evpProxyPrefix
+  }, getOnFinishGetCommitsToUpload(false))
 }
 
 module.exports = {

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js

@@ -1,9 +1,22 @@
 const request = require('../../exporters/common/request')
 const log = require('../../log')
+const {
+  incrementCountMetric,
+  distributionMetric,
+  TELEMETRY_ITR_SKIPPABLE_TESTS,
+  TELEMETRY_ITR_SKIPPABLE_TESTS_MS,
+  TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS,
+  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES,
+  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS,
+  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES,
+  getErrorTypeFromStatusCode
+} = require('../../ci-visibility/telemetry')
 
 function getSkippableSuites ({
   url,
   isEvpProxy,
+  evpProxyPrefix,
+  isGzipCompatible,
   env,
   service,
   repositoryUrl,
@@ -26,8 +39,12 @@ function getSkippableSuites ({
     url
   }
 
+  if (isGzipCompatible) {
+    options.headers['accept-encoding'] = 'gzip'
+  }
+
   if (isEvpProxy) {
-    options.path = '/evp_proxy/v2/api/v2/ci/tests/skippable'
+    options.path = `${evpProxyPrefix}/api/v2/ci/tests/skippable`
     options.headers['X-Datadog-EVP-Subdomain'] = 'api'
   } else {
     const apiKey = process.env.DATADOG_API_KEY || process.env.DD_API_KEY
@@ -59,13 +76,21 @@ function getSkippableSuites ({
     }
   })
 
-  request(data, options, (err, res) => {
+  incrementCountMetric(TELEMETRY_ITR_SKIPPABLE_TESTS)
+
+  const startTime = Date.now()
+
+  request(data, options, (err, res, statusCode) => {
+    distributionMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_MS, {}, Date.now() - startTime)
     if (err) {
+      const errorType = getErrorTypeFromStatusCode(statusCode)
+      incrementCountMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS, { errorType })
       done(err)
     } else {
       let skippableSuites = []
       try {
-        skippableSuites = JSON.parse(res)
+        const parsedResponse = JSON.parse(res)
+        skippableSuites = parsedResponse
           .data
           .filter(({ type }) => type === testLevel)
           .map(({ attributes: { suite, name } }) => {
@@ -74,8 +99,16 @@ function getSkippableSuites ({
             }
             return { suite, name }
           })
+        const { meta: { correlation_id: correlationId } } = parsedResponse
+        incrementCountMetric(
+          testLevel === 'test'
+            ? TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS : TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES,
+          {},
+          skippableSuites.length
+        )
+        distributionMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES, {}, res.length)
         log.debug(() => `Number of received skippable ${testLevel}s: ${skippableSuites.length}`)
-        done(null, skippableSuites)
+        done(null, skippableSuites, correlationId)
       } catch (err) {
         done(err)
       }