dd-trace 4.44.0 → 4.46.0

package/packages/dd-trace/src/appsec/blocking.js

@@ -9,6 +9,8 @@ let templateHtml = blockedTemplates.html
 let templateJson = blockedTemplates.json
 let templateGraphqlJson = blockedTemplates.graphqlJson
 
+let defaultBlockingActionParameters
+
 const responseBlockedSet = new WeakSet()
 
 const specificBlockingTypes = {
@@ -23,7 +25,7 @@ function addSpecificEndpoint (method, url, type) {
   detectedSpecificEndpoints[getSpecificKey(method, url)] = type
 }
 
-function getBlockWithRedirectData (rootSpan, actionParameters) {
+function getBlockWithRedirectData (actionParameters) {
   let statusCode = actionParameters.status_code
   if (!statusCode || statusCode < 300 || statusCode >= 400) {
     statusCode = 303
@@ -32,10 +34,6 @@ function getBlockWithRedirectData (rootSpan, actionParameters) {
     Location: actionParameters.location
   }
 
-  rootSpan.addTags({
-    'appsec.blocked': 'true'
-  })
-
   return { headers, statusCode }
 }
 
@@ -49,7 +47,7 @@ function getSpecificBlockingData (type) {
   }
 }
 
-function getBlockWithContentData (req, specificType, rootSpan, actionParameters) {
+function getBlockWithContentData (req, specificType, actionParameters) {
   let type
   let body
 
@@ -90,28 +88,28 @@ function getBlockWithContentData (req, specificType, rootSpan, actionParameters)
     'Content-Length': Buffer.byteLength(body)
   }
 
-  rootSpan.addTags({
-    'appsec.blocked': 'true'
-  })
-
   return { body, statusCode, headers }
 }
 
-function getBlockingData (req, specificType, rootSpan, actionParameters) {
+function getBlockingData (req, specificType, actionParameters) {
   if (actionParameters?.location) {
-    return getBlockWithRedirectData(rootSpan, actionParameters)
+    return getBlockWithRedirectData(actionParameters)
   } else {
-    return getBlockWithContentData(req, specificType, rootSpan, actionParameters)
+    return getBlockWithContentData(req, specificType, actionParameters)
   }
 }
 
-function block (req, res, rootSpan, abortController, actionParameters) {
+function block (req, res, rootSpan, abortController, actionParameters = defaultBlockingActionParameters) {
   if (res.headersSent) {
     log.warn('Cannot send blocking response when headers have already been sent')
     return
   }
 
-  const { body, headers, statusCode } = getBlockingData(req, null, rootSpan, actionParameters)
+  const { body, headers, statusCode } = getBlockingData(req, null, actionParameters)
+
+  rootSpan.addTags({
+    'appsec.blocked': 'true'
+  })
 
   for (const headerName of res.getHeaderNames()) {
     res.removeHeader(headerName)
@@ -125,7 +123,8 @@ function block (req, res, rootSpan, abortController, actionParameters) {
 }
 
 function getBlockingAction (actions) {
-  return actions?.block_request || actions?.redirect_request
+  // waf only returns one action, but it prioritizes redirect over block
+  return actions?.redirect_request || actions?.block_request
 }
 
 function setTemplates (config) {
@@ -152,6 +151,12 @@ function isBlocked (res) {
   return responseBlockedSet.has(res)
 }
 
+function setDefaultBlockingActionParameters (actions) {
+  const blockAction = actions?.find(action => action.id === 'block')
+
+  defaultBlockingActionParameters = blockAction?.parameters
+}
+
 module.exports = {
   addSpecificEndpoint,
   block,
@@ -159,5 +164,6 @@ module.exports = {
   getBlockingData,
   getBlockingAction,
   setTemplates,
-  isBlocked
+  isBlocked,
+  setDefaultBlockingActionParameters
 }

package/packages/dd-trace/src/appsec/graphql.js

@@ -94,11 +94,13 @@ function beforeWriteApolloGraphqlResponse ({ abortController, abortData }) {
     const rootSpan = web.root(req)
     if (!rootSpan) return
 
-    const blockingData = getBlockingData(req, specificBlockingTypes.GRAPHQL, rootSpan, requestData.wafAction)
+    const blockingData = getBlockingData(req, specificBlockingTypes.GRAPHQL, requestData.wafAction)
     abortData.statusCode = blockingData.statusCode
     abortData.headers = blockingData.headers
     abortData.message = blockingData.body
 
+    rootSpan.setTag('appsec.blocked', 'true')
+
     abortController?.abort()
   }
 

package/packages/dd-trace/src/appsec/iast/iast-log.js

@@ -78,7 +78,8 @@ const iastLog = {
 
   errorAndPublish (data) {
     this.error(data)
-    return this.publish(data, 'ERROR')
+    // publish is done automatically by log.error()
+    return this
   }
 }
 

package/packages/dd-trace/src/appsec/remote_config/manager.js

@@ -120,7 +120,10 @@ class RemoteConfigManager extends EventEmitter {
     const options = {
       url: this.url,
       method: 'POST',
-      path: '/v0.7/config'
+      path: '/v0.7/config',
+      headers: {
+        'Content-Type': 'application/json; charset=utf-8'
+      }
     }
 
     request(this.getPayload(), options, (err, data, statusCode) => {

package/packages/dd-trace/src/appsec/rule_manager.js

@@ -4,6 +4,8 @@ const fs = require('fs')
 const waf = require('./waf')
 const { ACKNOWLEDGED, ERROR } = require('./remote_config/apply_states')
 
+const blocking = require('./blocking')
+
 let defaultRules
 
 let appliedRulesData = new Map()
@@ -19,6 +21,8 @@ function loadRules (config) {
     : require('./recommended.json')
 
   waf.init(defaultRules, config)
+
+  blocking.setDefaultBlockingActionParameters(defaultRules?.actions)
 }
 
 function updateWafFromRC ({ toUnapply, toApply, toModify }) {
@@ -141,6 +145,8 @@ function updateWafFromRC ({ toUnapply, toApply, toModify }) {
       }
       if (newActions.modified) {
         appliedActions = newActions
+
+        blocking.setDefaultBlockingActionParameters(concatArrays(newActions))
       }
     } catch (err) {
       newApplyState = ERROR
@@ -242,6 +248,8 @@ function clearAllRules () {
   appliedExclusions.clear()
   appliedCustomRules.clear()
   appliedActions.clear()
+
+  blocking.setDefaultBlockingActionParameters(undefined)
 }
 
 module.exports = {

package/packages/dd-trace/src/appsec/telemetry.js

@@ -90,14 +90,14 @@ function updateRaspRequestsMetricTags (metrics, req, raspRuleType) {
   if (!enabled) return
 
   const tags = { rule_type: raspRuleType, waf_version: metrics.wafVersion }
-  appsecMetrics.count('appsec.rasp.rule.eval', tags).inc(1)
+  appsecMetrics.count('rasp.rule.eval', tags).inc(1)
 
   if (metrics.wafTimeout) {
-    appsecMetrics.count('appsec.rasp.timeout', tags).inc(1)
+    appsecMetrics.count('rasp.timeout', tags).inc(1)
   }
 
   if (metrics.ruleTriggered) {
-    appsecMetrics.count('appsec.rasp.rule.match', tags).inc(1)
+    appsecMetrics.count('rasp.rule.match', tags).inc(1)
   }
 }
 

package/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js

@@ -1,5 +1,30 @@
 const request = require('../../exporters/common/request')
 const id = require('../../id')
+const log = require('../../log')
+
+const {
+  incrementCountMetric,
+  distributionMetric,
+  TELEMETRY_KNOWN_TESTS,
+  TELEMETRY_KNOWN_TESTS_MS,
+  TELEMETRY_KNOWN_TESTS_ERRORS,
+  TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS,
+  TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES
+} = require('../../ci-visibility/telemetry')
+
+function getNumTests (knownTests) {
+  let totalNumTests = 0
+
+  for (const testModule of Object.values(knownTests)) {
+    for (const testSuite of Object.values(testModule)) {
+      for (const testList of Object.values(testSuite)) {
+        totalNumTests += testList.length
+      }
+    }
+  }
+
+  return totalNumTests
+}
 
 function getKnownTests ({
   url,
@@ -64,12 +89,26 @@ function getKnownTests ({
     }
   })
 
-  request(data, options, (err, res) => {
+  incrementCountMetric(TELEMETRY_KNOWN_TESTS)
+
+  const startTime = Date.now()
+
+  request(data, options, (err, res, statusCode) => {
+    distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
     if (err) {
+      incrementCountMetric(TELEMETRY_KNOWN_TESTS_ERRORS, { statusCode })
       done(err)
     } else {
       try {
         const { data: { attributes: { tests: knownTests } } } = JSON.parse(res)
+
+        const numTests = getNumTests(knownTests)
+
+        incrementCountMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS, {}, numTests)
+        distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES, {}, res.length)
+
+        log.debug(() => `Number of received known tests: ${numTests}`)
+
         done(null, knownTests)
       } catch (err) {
         done(err)

package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js

@@ -12,8 +12,7 @@ const {
   TELEMETRY_ENDPOINT_PAYLOAD_BYTES,
   TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS,
   TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
-  TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
-  getErrorTypeFromStatusCode
+  TELEMETRY_ENDPOINT_PAYLOAD_DROPPED
 } = require('../../../ci-visibility/telemetry')
 
 class Writer extends BaseWriter {
@@ -56,10 +55,9 @@ class Writer extends BaseWriter {
         Date.now() - startRequestTime
       )
       if (err) {
-        const errorType = getErrorTypeFromStatusCode(statusCode)
         incrementCountMetric(
           TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
-          { endpoint: 'code_coverage', errorType }
+          { endpoint: 'code_coverage', statusCode }
         )
         incrementCountMetric(
           TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -12,8 +12,7 @@ const {
   TELEMETRY_ENDPOINT_PAYLOAD_BYTES,
   TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS,
   TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
-  TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
-  getErrorTypeFromStatusCode
+  TELEMETRY_ENDPOINT_PAYLOAD_DROPPED
 } = require('../../../ci-visibility/telemetry')
 
 class Writer extends BaseWriter {
@@ -57,10 +56,9 @@ class Writer extends BaseWriter {
         Date.now() - startRequestTime
       )
       if (err) {
-        const errorType = getErrorTypeFromStatusCode(statusCode)
         incrementCountMetric(
           TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
-          { endpoint: 'test_cycle', errorType }
+          { endpoint: 'test_cycle', statusCode }
         )
         incrementCountMetric(
           TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -201,7 +201,8 @@ class CiVisibilityExporter extends AgentInfoExporter {
       isEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled && this._config.isEarlyFlakeDetectionEnabled,
       earlyFlakeDetectionNumRetries,
       earlyFlakeDetectionFaultyThreshold,
-      isFlakyTestRetriesEnabled
+      isFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled && this._config.isFlakyTestRetriesEnabled,
+      flakyTestRetriesCount: this._config.flakyTestRetriesCount
     }
   }
 

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -11,7 +11,8 @@ const {
   generatePackFilesForCommits,
   getCommitsRevList,
   isShallowRepository,
-  unshallowRepository
+  unshallowRepository,
+  isGitAvailable
 } = require('../../../plugins/util/git')
 
 const {
@@ -24,8 +25,7 @@ const {
   TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES,
   TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS,
   TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS,
-  TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES,
-  getErrorTypeFromStatusCode
+  TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES
 } = require('../../../ci-visibility/telemetry')
 
 const isValidSha1 = (sha) => /^[0-9a-f]{40}$/.test(sha)
@@ -92,8 +92,7 @@ function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy, ev
   request(localCommitData, options, (err, response, statusCode) => {
     distributionMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_MS, {}, Date.now() - startTime)
     if (err) {
-      const errorType = getErrorTypeFromStatusCode(statusCode)
-      incrementCountMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, { errorType })
+      incrementCountMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, { statusCode })
       const error = new Error(`Error fetching commits to exclude: ${err.message}`)
       return callback(error)
     }
@@ -178,8 +177,7 @@ function uploadPackFile ({ url, isEvpProxy, evpProxyPrefix, packFileToUpload, re
   request(form, options, (err, _, statusCode) => {
     distributionMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS, {}, Date.now() - startTime)
     if (err) {
-      const errorType = getErrorTypeFromStatusCode(statusCode)
-      incrementCountMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS, { errorType })
+      incrementCountMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS, { statusCode })
       const error = new Error(`Could not upload packfiles: status code ${statusCode}: ${err.message}`)
       return callback(error, uploadSize)
     }
@@ -245,6 +243,9 @@ function generateAndUploadPackFiles ({
  * This function uploads git metadata to CI Visibility's backend.
 */
 function sendGitMetadata (url, { isEvpProxy, evpProxyPrefix }, configRepositoryUrl, callback) {
+  if (!isGitAvailable()) {
+    return callback(new Error('Git is not available'))
+  }
   let repositoryUrl = configRepositoryUrl
   if (!repositoryUrl) {
     repositoryUrl = getRepositoryUrl()

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js

@@ -8,8 +8,7 @@ const {
   TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS,
   TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES,
   TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS,
-  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES,
-  getErrorTypeFromStatusCode
+  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES
 } = require('../../ci-visibility/telemetry')
 
 function getSkippableSuites ({
@@ -83,8 +82,7 @@ function getSkippableSuites ({
   request(data, options, (err, res, statusCode) => {
     distributionMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_MS, {}, Date.now() - startTime)
     if (err) {
-      const errorType = getErrorTypeFromStatusCode(statusCode)
-      incrementCountMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS, { errorType })
+      incrementCountMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS, { statusCode })
       done(err)
     } else {
       let skippableSuites = []

package/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js

@@ -7,8 +7,7 @@ const {
   TELEMETRY_GIT_REQUESTS_SETTINGS,
   TELEMETRY_GIT_REQUESTS_SETTINGS_MS,
   TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS,
-  TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE,
-  getErrorTypeFromStatusCode
+  TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE
 } = require('../telemetry')
 
 const DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES = 2
@@ -81,8 +80,7 @@ function getLibraryConfiguration ({
   request(data, options, (err, res, statusCode) => {
     distributionMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_MS, {}, Date.now() - startTime)
     if (err) {
-      const errorType = getErrorTypeFromStatusCode(statusCode)
-      incrementCountMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS, { errorType })
+      incrementCountMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS, { statusCode })
       done(err)
     } else {
       try {

package/packages/dd-trace/src/ci-visibility/telemetry.js

@@ -10,13 +10,24 @@ const formattedTags = {
   isCodeCoverageEnabled: 'coverage_enabled',
   isSuitesSkippingEnabled: 'itrskip_enabled',
   hasCodeOwners: 'has_code_owners',
-  isUnsupportedCIProvider: 'is_unsupported_ci'
+  isUnsupportedCIProvider: 'is_unsupported_ci',
+  isNew: 'is_new',
+  isRum: 'is_rum',
+  browserDriver: 'browser_driver'
 }
 
 // Transform tags dictionary to array of strings.
 // If tag value is true, then only tag key is added to the array.
 function formatMetricTags (tagsDictionary) {
   return Object.keys(tagsDictionary).reduce((acc, tagKey) => {
+    if (tagKey === 'statusCode') {
+      const statusCode = tagsDictionary[tagKey]
+      if (isStatusCode400(statusCode)) {
+        acc.push(`status_code:${statusCode}`)
+      }
+      acc.push(`error_type:${getErrorTypeFromStatusCode(statusCode)}`)
+      return acc
+    }
     const formattedTagKey = formattedTags[tagKey] || tagKey
     if (tagsDictionary[tagKey] === true) {
       acc.push(formattedTagKey)
@@ -36,6 +47,7 @@ function distributionMetric (name, tags, measure) {
 }
 
 // CI Visibility telemetry events
+const TELEMETRY_TEST_SESSION = 'test_session'
 const TELEMETRY_EVENT_CREATED = 'event_created'
 const TELEMETRY_EVENT_FINISHED = 'event_finished'
 const TELEMETRY_CODE_COVERAGE_STARTED = 'code_coverage_started'
@@ -74,6 +86,16 @@ const TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS = 'itr_skippable_tests.request_errors
 const TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES = 'itr_skippable_tests.response_suites'
 const TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS = 'itr_skippable_tests.response_tests'
 const TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES = 'itr_skippable_tests.response_bytes'
+// early flake detection
+const TELEMETRY_KNOWN_TESTS = 'early_flake_detection.request'
+const TELEMETRY_KNOWN_TESTS_MS = 'early_flake_detection.request_ms'
+const TELEMETRY_KNOWN_TESTS_ERRORS = 'early_flake_detection.request_errors'
+const TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS = 'early_flake_detection.response_tests'
+const TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES = 'early_flake_detection.response_bytes'
+
+function isStatusCode400 (statusCode) {
+  return statusCode >= 400 && statusCode < 500
+}
 
 function getErrorTypeFromStatusCode (statusCode) {
   if (statusCode >= 400 && statusCode < 500) {
@@ -88,6 +110,7 @@ function getErrorTypeFromStatusCode (statusCode) {
 module.exports = {
   incrementCountMetric,
   distributionMetric,
+  TELEMETRY_TEST_SESSION,
   TELEMETRY_EVENT_CREATED,
   TELEMETRY_EVENT_FINISHED,
   TELEMETRY_CODE_COVERAGE_STARTED,
@@ -126,5 +149,9 @@ module.exports = {
   TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES,
   TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS,
   TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES,
-  getErrorTypeFromStatusCode
+  TELEMETRY_KNOWN_TESTS,
+  TELEMETRY_KNOWN_TESTS_MS,
+  TELEMETRY_KNOWN_TESTS_ERRORS,
+  TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS,
+  TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES
 }