dd-trace 5.19.0 → 5.21.0

package/packages/dd-trace/src/appsec/recommended.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.12.0"
+    "rules_version": "1.13.0"
   },
   "rules": [
     {
@@ -6285,6 +6285,55 @@
         "stack_trace"
       ]
     },
+    {
+      "id": "rasp-932-100",
+      "name": "Shell injection exploit",
+      "enabled": false,
+      "tags": {
+        "type": "command_injection",
+        "category": "vulnerability_trigger",
+        "cwe": "77",
+        "capec": "1000/152/248/88",
+        "confidence": "0",
+        "module": "rasp"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "resource": [
+              {
+                "address": "server.sys.shell.cmd"
+              }
+            ],
+            "params": [
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              },
+              {
+                "address": "grpc.server.request.message"
+              },
+              {
+                "address": "graphql.server.all_resolvers"
+              },
+              {
+                "address": "graphql.server.resolver"
+              }
+            ]
+          },
+          "operator": "shi_detector"
+        }
+      ],
+      "transformers": [],
+      "on_match": [
+        "stack_trace"
+      ]
+    },
     {
       "id": "rasp-934-100",
       "name": "Server-side request forgery exploit",
@@ -8388,6 +8437,57 @@
     }
   ],
   "processors": [
+    {
+      "id": "http-endpoint-fingerprint",
+      "generator": "http_endpoint_fingerprint",
+      "conditions": [
+        {
+          "operator": "exists",
+          "parameters": {
+            "inputs": [
+              {
+                "address": "waf.context.event"
+              },
+              {
+                "address": "server.business_logic.users.login.failure"
+              },
+              {
+                "address": "server.business_logic.users.login.success"
+              }
+            ]
+          }
+        }
+      ],
+      "parameters": {
+        "mappings": [
+          {
+            "method": [
+              {
+                "address": "server.request.method"
+              }
+            ],
+            "uri_raw": [
+              {
+                "address": "server.request.uri.raw"
+              }
+            ],
+            "body": [
+              {
+                "address": "server.request.body"
+              }
+            ],
+            "query": [
+              {
+                "address": "server.request.query"
+              }
+            ],
+            "output": "_dd.appsec.fp.http.endpoint"
+          }
+        ]
+      },
+      "evaluate": false,
+      "output": true
+    },
     {
       "id": "extract-content",
       "generator": "extract_schema",
@@ -8537,6 +8637,124 @@
       },
       "evaluate": false,
       "output": true
+    },
+    {
+      "id": "http-header-fingerprint",
+      "generator": "http_header_fingerprint",
+      "conditions": [
+        {
+          "operator": "exists",
+          "parameters": {
+            "inputs": [
+              {
+                "address": "waf.context.event"
+              },
+              {
+                "address": "server.business_logic.users.login.failure"
+              },
+              {
+                "address": "server.business_logic.users.login.success"
+              }
+            ]
+          }
+        }
+      ],
+      "parameters": {
+        "mappings": [
+          {
+            "headers": [
+              {
+                "address": "server.request.headers.no_cookies"
+              }
+            ],
+            "output": "_dd.appsec.fp.http.header"
+          }
+        ]
+      },
+      "evaluate": false,
+      "output": true
+    },
+    {
+      "id": "http-network-fingerprint",
+      "generator": "http_network_fingerprint",
+      "conditions": [
+        {
+          "operator": "exists",
+          "parameters": {
+            "inputs": [
+              {
+                "address": "waf.context.event"
+              },
+              {
+                "address": "server.business_logic.users.login.failure"
+              },
+              {
+                "address": "server.business_logic.users.login.success"
+              }
+            ]
+          }
+        }
+      ],
+      "parameters": {
+        "mappings": [
+          {
+            "headers": [
+              {
+                "address": "server.request.headers.no_cookies"
+              }
+            ],
+            "output": "_dd.appsec.fp.http.network"
+          }
+        ]
+      },
+      "evaluate": false,
+      "output": true
+    },
+    {
+      "id": "session-fingerprint",
+      "generator": "session_fingerprint",
+      "conditions": [
+        {
+          "operator": "exists",
+          "parameters": {
+            "inputs": [
+              {
+                "address": "waf.context.event"
+              },
+              {
+                "address": "server.business_logic.users.login.failure"
+              },
+              {
+                "address": "server.business_logic.users.login.success"
+              }
+            ]
+          }
+        }
+      ],
+      "parameters": {
+        "mappings": [
+          {
+            "cookies": [
+              {
+                "address": "server.request.cookies"
+              }
+            ],
+            "session_id": [
+              {
+                "address": "usr.session_id"
+              }
+            ],
+            "user_id": [
+              {
+                "address": "usr.id"
+              }
+            ],
+            "output": "_dd.appsec.fp.session"
+          }
+        ]
+      },
+      "evaluate": false,
+      "output": true
     }
   ],
   "scanners": [
@@ -9562,4 +9780,4 @@
       }
     }
   ]
-}
+}

package/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js

@@ -1,5 +1,30 @@
 const request = require('../../exporters/common/request')
 const id = require('../../id')
+const log = require('../../log')
+
+const {
+  incrementCountMetric,
+  distributionMetric,
+  TELEMETRY_KNOWN_TESTS,
+  TELEMETRY_KNOWN_TESTS_MS,
+  TELEMETRY_KNOWN_TESTS_ERRORS,
+  TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS,
+  TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES
+} = require('../../ci-visibility/telemetry')
+
+function getNumTests (knownTests) {
+  let totalNumTests = 0
+
+  for (const testModule of Object.values(knownTests)) {
+    for (const testSuite of Object.values(testModule)) {
+      for (const testList of Object.values(testSuite)) {
+        totalNumTests += testList.length
+      }
+    }
+  }
+
+  return totalNumTests
+}
 
 function getKnownTests ({
   url,
@@ -64,12 +89,26 @@ function getKnownTests ({
     }
   })
 
-  request(data, options, (err, res) => {
+  incrementCountMetric(TELEMETRY_KNOWN_TESTS)
+
+  const startTime = Date.now()
+
+  request(data, options, (err, res, statusCode) => {
+    distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
     if (err) {
+      incrementCountMetric(TELEMETRY_KNOWN_TESTS_ERRORS, { statusCode })
       done(err)
     } else {
       try {
         const { data: { attributes: { tests: knownTests } } } = JSON.parse(res)
+
+        const numTests = getNumTests(knownTests)
+
+        incrementCountMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS, {}, numTests)
+        distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES, {}, res.length)
+
+        log.debug(() => `Number of received known tests: ${numTests}`)
+
         done(null, knownTests)
       } catch (err) {
         done(err)

package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js

@@ -12,8 +12,7 @@ const {
   TELEMETRY_ENDPOINT_PAYLOAD_BYTES,
   TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS,
   TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
-  TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
-  getErrorTypeFromStatusCode
+  TELEMETRY_ENDPOINT_PAYLOAD_DROPPED
 } = require('../../../ci-visibility/telemetry')
 
 class Writer extends BaseWriter {
@@ -56,10 +55,9 @@ class Writer extends BaseWriter {
         Date.now() - startRequestTime
       )
       if (err) {
-        const errorType = getErrorTypeFromStatusCode(statusCode)
         incrementCountMetric(
           TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
-          { endpoint: 'code_coverage', errorType }
+          { endpoint: 'code_coverage', statusCode }
         )
         incrementCountMetric(
           TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -12,8 +12,7 @@ const {
   TELEMETRY_ENDPOINT_PAYLOAD_BYTES,
   TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS,
   TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
-  TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
-  getErrorTypeFromStatusCode
+  TELEMETRY_ENDPOINT_PAYLOAD_DROPPED
 } = require('../../../ci-visibility/telemetry')
 
 class Writer extends BaseWriter {
@@ -57,10 +56,9 @@ class Writer extends BaseWriter {
         Date.now() - startRequestTime
       )
       if (err) {
-        const errorType = getErrorTypeFromStatusCode(statusCode)
         incrementCountMetric(
           TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
-          { endpoint: 'test_cycle', errorType }
+          { endpoint: 'test_cycle', statusCode }
         )
         incrementCountMetric(
           TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -11,7 +11,8 @@ const {
   generatePackFilesForCommits,
   getCommitsRevList,
   isShallowRepository,
-  unshallowRepository
+  unshallowRepository,
+  isGitAvailable
 } = require('../../../plugins/util/git')
 
 const {
@@ -24,8 +25,7 @@ const {
   TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES,
   TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS,
   TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS,
-  TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES,
-  getErrorTypeFromStatusCode
+  TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES
 } = require('../../../ci-visibility/telemetry')
 
 const isValidSha1 = (sha) => /^[0-9a-f]{40}$/.test(sha)
@@ -92,8 +92,7 @@ function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy, ev
   request(localCommitData, options, (err, response, statusCode) => {
     distributionMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_MS, {}, Date.now() - startTime)
     if (err) {
-      const errorType = getErrorTypeFromStatusCode(statusCode)
-      incrementCountMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, { errorType })
+      incrementCountMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, { statusCode })
       const error = new Error(`Error fetching commits to exclude: ${err.message}`)
       return callback(error)
     }
@@ -178,8 +177,7 @@ function uploadPackFile ({ url, isEvpProxy, evpProxyPrefix, packFileToUpload, re
   request(form, options, (err, _, statusCode) => {
     distributionMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS, {}, Date.now() - startTime)
     if (err) {
-      const errorType = getErrorTypeFromStatusCode(statusCode)
-      incrementCountMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS, { errorType })
+      incrementCountMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS, { statusCode })
       const error = new Error(`Could not upload packfiles: status code ${statusCode}: ${err.message}`)
       return callback(error, uploadSize)
     }
@@ -245,6 +243,9 @@ function generateAndUploadPackFiles ({
  * This function uploads git metadata to CI Visibility's backend.
 */
 function sendGitMetadata (url, { isEvpProxy, evpProxyPrefix }, configRepositoryUrl, callback) {
+  if (!isGitAvailable()) {
+    return callback(new Error('Git is not available'))
+  }
   let repositoryUrl = configRepositoryUrl
   if (!repositoryUrl) {
     repositoryUrl = getRepositoryUrl()

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js

@@ -8,8 +8,7 @@ const {
   TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS,
   TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES,
   TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS,
-  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES,
-  getErrorTypeFromStatusCode
+  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES
 } = require('../../ci-visibility/telemetry')
 
 function getSkippableSuites ({
@@ -83,8 +82,7 @@ function getSkippableSuites ({
   request(data, options, (err, res, statusCode) => {
     distributionMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_MS, {}, Date.now() - startTime)
     if (err) {
-      const errorType = getErrorTypeFromStatusCode(statusCode)
-      incrementCountMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS, { errorType })
+      incrementCountMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS, { statusCode })
       done(err)
     } else {
       let skippableSuites = []

package/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js

@@ -7,8 +7,7 @@ const {
   TELEMETRY_GIT_REQUESTS_SETTINGS,
   TELEMETRY_GIT_REQUESTS_SETTINGS_MS,
   TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS,
-  TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE,
-  getErrorTypeFromStatusCode
+  TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE
 } = require('../telemetry')
 
 const DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES = 2
@@ -81,8 +80,7 @@ function getLibraryConfiguration ({
   request(data, options, (err, res, statusCode) => {
     distributionMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_MS, {}, Date.now() - startTime)
     if (err) {
-      const errorType = getErrorTypeFromStatusCode(statusCode)
-      incrementCountMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS, { errorType })
+      incrementCountMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS, { statusCode })
       done(err)
     } else {
       try {

package/packages/dd-trace/src/ci-visibility/telemetry.js

@@ -10,13 +10,24 @@ const formattedTags = {
   isCodeCoverageEnabled: 'coverage_enabled',
   isSuitesSkippingEnabled: 'itrskip_enabled',
   hasCodeOwners: 'has_code_owners',
-  isUnsupportedCIProvider: 'is_unsupported_ci'
+  isUnsupportedCIProvider: 'is_unsupported_ci',
+  isNew: 'is_new',
+  isRum: 'is_rum',
+  browserDriver: 'browser_driver'
 }
 
 // Transform tags dictionary to array of strings.
 // If tag value is true, then only tag key is added to the array.
 function formatMetricTags (tagsDictionary) {
   return Object.keys(tagsDictionary).reduce((acc, tagKey) => {
+    if (tagKey === 'statusCode') {
+      const statusCode = tagsDictionary[tagKey]
+      if (isStatusCode400(statusCode)) {
+        acc.push(`status_code:${statusCode}`)
+      }
+      acc.push(`error_type:${getErrorTypeFromStatusCode(statusCode)}`)
+      return acc
+    }
     const formattedTagKey = formattedTags[tagKey] || tagKey
     if (tagsDictionary[tagKey] === true) {
       acc.push(formattedTagKey)
@@ -36,6 +47,7 @@ function distributionMetric (name, tags, measure) {
 }
 
 // CI Visibility telemetry events
+const TELEMETRY_TEST_SESSION = 'test_session'
 const TELEMETRY_EVENT_CREATED = 'event_created'
 const TELEMETRY_EVENT_FINISHED = 'event_finished'
 const TELEMETRY_CODE_COVERAGE_STARTED = 'code_coverage_started'
@@ -74,6 +86,16 @@ const TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS = 'itr_skippable_tests.request_errors
 const TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES = 'itr_skippable_tests.response_suites'
 const TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS = 'itr_skippable_tests.response_tests'
 const TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES = 'itr_skippable_tests.response_bytes'
+// early flake detection
+const TELEMETRY_KNOWN_TESTS = 'early_flake_detection.request'
+const TELEMETRY_KNOWN_TESTS_MS = 'early_flake_detection.request_ms'
+const TELEMETRY_KNOWN_TESTS_ERRORS = 'early_flake_detection.request_errors'
+const TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS = 'early_flake_detection.response_tests'
+const TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES = 'early_flake_detection.response_bytes'
+
+function isStatusCode400 (statusCode) {
+  return statusCode >= 400 && statusCode < 500
+}
 
 function getErrorTypeFromStatusCode (statusCode) {
   if (statusCode >= 400 && statusCode < 500) {
@@ -88,6 +110,7 @@ function getErrorTypeFromStatusCode (statusCode) {
 module.exports = {
   incrementCountMetric,
   distributionMetric,
+  TELEMETRY_TEST_SESSION,
   TELEMETRY_EVENT_CREATED,
   TELEMETRY_EVENT_FINISHED,
   TELEMETRY_CODE_COVERAGE_STARTED,
@@ -126,5 +149,9 @@ module.exports = {
   TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES,
   TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS,
   TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES,
-  getErrorTypeFromStatusCode
+  TELEMETRY_KNOWN_TESTS,
+  TELEMETRY_KNOWN_TESTS_MS,
+  TELEMETRY_KNOWN_TESTS_ERRORS,
+  TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS,
+  TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES
 }