dd-trace 3.14.0 → 3.15.0

package/packages/dd-trace/src/appsec/recommended.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.5.0"
+    "rules_version": "1.5.2"
   },
   "rules": [
     {
@@ -199,33 +199,6 @@
         "lowercase"
       ]
     },
-    {
-      "id": "crs-921-140",
-      "name": "HTTP Header Injection Attack via headers",
-      "tags": {
-        "type": "http_protocol_violation",
-        "crs_id": "921140",
-        "category": "attack_attempt"
-      },
-      "conditions": [
-        {
-          "parameters": {
-            "inputs": [
-              {
-                "address": "server.request.headers.no_cookies"
-              }
-            ],
-            "regex": "[\\n\\r]",
-            "options": {
-              "case_sensitive": true,
-              "min_length": 1
-            }
-          },
-          "operator": "match_regex"
-        }
-      ],
-      "transformers": []
-    },
     {
       "id": "crs-921-160",
       "name": "HTTP Header Injection Attack via payload (CR/LF and header-name detected)",
@@ -245,7 +218,7 @@
                 "address": "server.request.path_params"
               }
             ],
-            "regex": "[\\n\\r]+(?:\\s|location|refresh|(?:set-)?cookie|(?:x-)?(?:forwarded-(?:for|host|server)|host|via|remote-ip|remote-addr|originating-IP))\\s*:",
+            "regex": "[\\n\\r]+(?:refresh|(?:set-)?cookie|(?:x-)?(?:forwarded-(?:for|host|server)|via|remote-ip|remote-addr|originating-IP))\\s*:",
             "options": {
               "case_sensitive": true,
               "min_length": 3
@@ -278,7 +251,7 @@
                 "address": "server.request.headers.no_cookies"
               }
             ],
-            "regex": "(?:%(?:c(?:0%(?:[2aq]f|5c|9v)|1%(?:[19p]c|8s|af))|2(?:5(?:c(?:0%25af|1%259c)|2f|5c)|%46|f)|(?:(?:f(?:8%8)?0%8|e)0%80%a|bg%q)f|%3(?:2(?:%(?:%6|4)6|F)|5%%63)|u(?:221[56]|002f|EFC8|F025)|1u|5c)|0x(?:2f|5c)|\\/|\\x5c)(?:%(?:(?:f(?:(?:c%80|8)%8)?0%8|e)0%80%ae|2(?:(?:5(?:c0%25a|2))?e|%45)|u(?:(?:002|ff0)e|2024)|%32(?:%(?:%6|4)5|E)|c0(?:%[256aef]e|\\.))|\\.(?:%0[01]|\\?)?|\\?\\.?|0x2e){2,3}(?:%(?:c(?:0%(?:[2aq]f|5c|9v)|1%(?:[19p]c|8s|af))|2(?:5(?:c(?:0%25af|1%259c)|2f|5c)|%46|f)|(?:(?:f(?:8%8)?0%8|e)0%80%a|bg%q)f|%3(?:2(?:%(?:%6|4)6|F)|5%%63)|u(?:221[56]|002f|EFC8|F025)|1u|5c)|0x(?:2f|5c)|\\/|\\x5c)",
+            "regex": "(?:%(?:c(?:0%(?:[2aq]f|5c|9v)|1%(?:[19p]c|8s|af))|2(?:5(?:c(?:0%25af|1%259c)|2f|5c)|%46|f)|(?:(?:f(?:8%8)?0%8|e)0%80%a|bg%q)f|%3(?:2(?:%(?:%6|4)6|F)|5%%63)|u(?:221[56]|002f|EFC8|F025)|1u|5c)|0x(?:2f|5c)|\\/|\\x5c)(?:%(?:(?:f(?:(?:c%80|8)%8)?0%8|e)0%80%ae|2(?:(?:5(?:c0%25a|2))?e|%45)|u(?:(?:002|ff0)e|2024)|%32(?:%(?:%6|4)5|E)|c0(?:%[256aef]e|\\.))|\\.(?:%0[01])?|0x2e){2,3}(?:%(?:c(?:0%(?:[2aq]f|5c|9v)|1%(?:[19p]c|8s|af))|2(?:5(?:c(?:0%25af|1%259c)|2f|5c)|%46|f)|(?:(?:f(?:8%8)?0%8|e)0%80%a|bg%q)f|%3(?:2(?:%(?:%6|4)6|F)|5%%63)|u(?:221[56]|002f|EFC8|F025)|1u|5c)|0x(?:2f|5c)|\\/|\\x5c)",
             "options": {
               "min_length": 4
             }
@@ -1378,16 +1351,11 @@
               "etc/timezone",
               "etc/modules",
               "etc/passwd",
-              "etc/passwd~",
-              "etc/passwd-",
               "etc/shadow",
-              "etc/shadow~",
-              "etc/shadow-",
               "etc/fstab",
               "etc/motd",
               "etc/hosts",
               "etc/group",
-              "etc/group-",
               "etc/alias",
               "etc/crontab",
               "etc/crypttab",
@@ -1834,7 +1802,7 @@
                 "address": "server.request.path_params"
               }
             ],
-            "regex": "^(?i:file|ftps?|http)://.*?\\?+$",
+            "regex": "^(?i:file|ftps?)://.*?\\?+$",
             "options": {
               "case_sensitive": true,
               "min_length": 4
@@ -1898,11 +1866,8 @@
               "dev/tcp/",
               "dev/udp/",
               "dev/zero",
-              "etc/group",
               "etc/master.passwd",
-              "etc/passwd",
               "etc/pwd.db",
-              "etc/shadow",
               "etc/shells",
               "etc/spwd.db",
               "proc/self/",
@@ -4117,9 +4082,7 @@
               "java.lang.number",
               "java.lang.object",
               "java.lang.process",
-              "java.lang.processbuilder",
               "java.lang.reflect",
-              "java.lang.runtime",
               "java.lang.string",
               "java.lang.stringbuilder",
               "java.lang.system",
@@ -4452,6 +4415,74 @@
       ],
       "transformers": []
     },
+    {
+      "id": "dog-934-001",
+      "name": "XXE - XML file loads external entity",
+      "tags": {
+        "type": "xxe",
+        "category": "attack_attempt",
+        "confidence": "0"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "grpc.server.request.message"
+              }
+            ],
+            "regex": "(?:<\\?xml[^>]*>.*)<!ENTITY[^>]+SYSTEM\\s+[^>]+>",
+            "options": {
+              "case_sensitive": false,
+              "min_length": 24
+            }
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": []
+    },
+    {
+      "id": "dog-942-001",
+      "name": "Blind XSS callback domains",
+      "tags": {
+        "type": "xss",
+        "category": "attack_attempt",
+        "confidence": "1"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              },
+              {
+                "address": "server.request.headers.no_cookies"
+              },
+              {
+                "address": "grpc.server.request.message"
+              }
+            ],
+            "regex": "https?:\\/\\/(?:.*\\.)?(?:bxss\\.in|xss\\.ht|js\\.rip)",
+            "options": {
+              "case_sensitive": false
+            }
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": []
+    },
     {
       "id": "nfd-000-001",
       "name": "Detect common directory discovery scans",
@@ -5080,36 +5111,6 @@
         "removeNulls"
       ]
     },
-    {
-      "id": "sqr-000-007",
-      "name": "NoSQL: Detect common exploitation strategy",
-      "tags": {
-        "type": "nosql_injection",
-        "category": "attack_attempt"
-      },
-      "conditions": [
-        {
-          "parameters": {
-            "inputs": [
-              {
-                "address": "server.request.query"
-              },
-              {
-                "address": "server.request.body"
-              },
-              {
-                "address": "server.request.path_params"
-              }
-            ],
-            "regex": "^\\$(eq|ne|(l|g)te?|n?in|not|(n|x|)or|and|regex|where|expr|exists)$"
-          },
-          "operator": "match_regex"
-        }
-      ],
-      "transformers": [
-        "keys_only"
-      ]
-    },
     {
       "id": "sqr-000-008",
       "name": "Windows: Detect attempts to exfiltrate .ini files",
@@ -5275,7 +5276,7 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "^(jar:)?(http|https):\\/\\/([0-9oq]{1,5}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|[0-9]{1,10})(:[0-9]{1,5})?(\\/.*|)$"
+            "regex": "^(jar:)?(http|https):\\/\\/([0-9oq]{1,5}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|[0-9]{1,10})(:[0-9]{1,5})?(\\/[^:@]*)?$"
           },
           "operator": "match_regex"
         }
@@ -5309,7 +5310,7 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "^(jar:)?(http|https):\\/\\/((\\[)?[:0-9a-f\\.x]{2,}(\\])?)(:[0-9]{1,5})?(\\/.*)?$"
+            "regex": "^(jar:)?(http|https):\\/\\/((\\[)?[:0-9a-f\\.x]{2,}(\\])?)(:[0-9]{1,5})?(\\/[^:@]*)?$"
           },
           "operator": "match_regex"
         }
@@ -5346,7 +5347,7 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "(http|https):\\/\\/(?:.*\\.)?(?:burpcollaborator\\.net|localtest\\.me|mail\\.ebc\\.apple\\.com|bugbounty\\.dod\\.network|.*\\.[nx]ip\\.io|oastify\\.com|oast\\.(?:pro|live|site|online|fun|me)|sslip\\.io|requestbin\\.com|requestbin\\.net|hookbin\\.com|webhook\\.site|canarytokens\\.com|interact\\.sh|ngrok\\.io|bugbounty\\.click)"
+            "regex": "(http|https):\\/\\/(?:.*\\.)?(?:burpcollaborator\\.net|localtest\\.me|mail\\.ebc\\.apple\\.com|bugbounty\\.dod\\.network|.*\\.[nx]ip\\.io|oastify\\.com|oast\\.(?:pro|live|site|online|fun|me)|sslip\\.io|requestbin\\.com|requestbin\\.net|hookbin\\.com|webhook\\.site|canarytokens\\.com|interact\\.sh|ngrok\\.io|bugbounty\\.click|prbly\\.win|qualysperiscope\\.com)"
           },
           "operator": "match_regex"
         }
@@ -6720,4 +6721,4 @@
       "transformers": []
     }
   ]
-}
+}

package/packages/dd-trace/src/appsec/remote_config/capabilities.js

@@ -3,5 +3,6 @@
 module.exports = {
   ASM_ACTIVATION: 1n << 1n,
   ASM_IP_BLOCKING: 1n << 2n,
-  ASM_DD_RULES: 1n << 3n
+  ASM_DD_RULES: 1n << 3n,
+  ASM_USER_BLOCKING: 1n << 7n
 }

package/packages/dd-trace/src/appsec/remote_config/index.js

@@ -35,12 +35,15 @@ function enable (config) {
 function enableAsmData (appsecConfig) {
   if (rc && appsecConfig && appsecConfig.rules === undefined) {
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_IP_BLOCKING, true)
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_USER_BLOCKING, true)
     rc.on('ASM_DATA', _asmDataListener)
   }
 }
 
 function disableAsmData () {
   if (rc) {
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_IP_BLOCKING, false)
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_USER_BLOCKING, false)
     rc.off('ASM_DATA', _asmDataListener)
   }
 }

package/packages/dd-trace/src/appsec/sdk/index.js

@@ -1,10 +1,16 @@
 'use strict'
 
 const { trackUserLoginSuccessEvent, trackUserLoginFailureEvent, trackCustomEvent } = require('./track_event')
+const { checkUserAndSetUser, blockRequest } = require('./user_blocking')
+const { loadTemplates } = require('../blocking')
+const { setUser } = require('./set_user')
 
 class AppsecSdk {
-  constructor (tracer) {
+  constructor (tracer, config) {
     this._tracer = tracer
+    if (config) {
+      loadTemplates(config)
+    }
   }
 
   trackUserLoginSuccessEvent (user, metadata) {
@@ -18,6 +24,18 @@ class AppsecSdk {
   trackCustomEvent (eventName, metadata) {
     return trackCustomEvent(this._tracer, eventName, metadata)
   }
+
+  isUserBlocked (user) {
+    return checkUserAndSetUser(this._tracer, user)
+  }
+
+  blockRequest (req, res) {
+    return blockRequest(this._tracer, req, res)
+  }
+
+  setUser (user) {
+    return setUser(this._tracer, user)
+  }
 }
 
 module.exports = AppsecSdk

package/packages/dd-trace/src/appsec/sdk/noop.js

@@ -6,6 +6,12 @@ class NoopAppsecSdk {
   trackUserLoginFailureEvent () {}
 
   trackCustomEvent () {}
+
+  isUserBlocked () {}
+
+  blockRequest () {}
+
+  setUser () {}
 }
 
 module.exports = NoopAppsecSdk

package/packages/dd-trace/src/appsec/sdk/set_user.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const { getRootSpan } = require('./utils')
+const log = require('../../log')
+
+function setUserTags (user, rootSpan) {
+  for (const k of Object.keys(user)) {
+    rootSpan.setTag(`usr.${k}`, '' + user[k])
+  }
+}
+
+function setUser (tracer, user) {
+  if (!user || !user.id) {
+    log.warn('Invalid user provided to setUser')
+    return
+  }
+
+  const rootSpan = getRootSpan(tracer)
+  if (!rootSpan) {
+    log.warn('Root span not available in setUser')
+    return
+  }
+
+  setUserTags(user, rootSpan)
+}
+
+module.exports = {
+  setUserTags,
+  setUser
+}

package/packages/dd-trace/src/appsec/sdk/track_event.js

@@ -3,6 +3,7 @@
 const log = require('../../log')
 const { getRootSpan } = require('./utils')
 const { MANUAL_KEEP } = require('../../../../../ext/tags')
+const { setUserTags } = require('./set_user')
 
 function trackUserLoginSuccessEvent (tracer, user, metadata) {
   // TODO: better user check here and in _setUser() ?
@@ -17,8 +18,7 @@ function trackUserLoginSuccessEvent (tracer, user, metadata) {
     return
   }
 
-  // TODO: use sdk._setUser(user, rootSpan) (available in User Blocking PR #2710)
-  tracer.setUser(user)
+  setUserTags(user, rootSpan)
 
   trackEvent(tracer, 'users.login.success', metadata, 'trackUserLoginSuccessEvent', rootSpan)
 }

package/packages/dd-trace/src/appsec/sdk/user_blocking.js

@@ -0,0 +1,73 @@
+'use strict'
+
+const { USER_ID } = require('../addresses')
+const Gateway = require('../gateway/engine')
+const { getRootSpan } = require('./utils')
+const { block } = require('../blocking')
+const { storage } = require('../../../../datadog-core')
+const { setUserTags } = require('./set_user')
+const log = require('../../log')
+
+function isUserBlocked (user) {
+  const results = Gateway.propagate({ [USER_ID]: user.id })
+
+  if (!results) {
+    return false
+  }
+
+  for (const entry of results) {
+    if (entry && entry.includes('block')) {
+      return true
+    }
+  }
+
+  return false
+}
+
+function checkUserAndSetUser (tracer, user) {
+  if (!user || !user.id) {
+    log.warn('Invalid user provided to isUserBlocked')
+    return false
+  }
+
+  const rootSpan = getRootSpan(tracer)
+  if (rootSpan) {
+    if (!rootSpan.context()._tags['usr.id']) {
+      setUserTags(user, rootSpan)
+    }
+  } else {
+    log.warn('Root span not available in isUserBlocked')
+  }
+
+  return isUserBlocked(user)
+}
+
+function blockRequest (tracer, req, res) {
+  if (!req || !res) {
+    const store = storage.getStore()
+    if (store) {
+      req = req || store.req
+      res = res || store.res
+    }
+  }
+
+  if (!req || !res) {
+    log.warn('Requests or response object not available in blockRequest')
+    return false
+  }
+
+  const rootSpan = getRootSpan(tracer)
+  if (!rootSpan) {
+    log.warn('Root span not available in blockRequest')
+    return false
+  }
+
+  block(req, res, rootSpan)
+
+  return true
+}
+
+module.exports = {
+  checkUserAndSetUser,
+  blockRequest
+}

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -8,6 +8,19 @@ const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligen
 const log = require('../../log')
 const AgentInfoExporter = require('../../exporters/common/agent-info-exporter')
 
+function getTestConfigurationTags (tags) {
+  if (!tags) {
+    return {}
+  }
+  return Object.keys(tags).reduce((acc, key) => {
+    if (key.startsWith('test.configuration.')) {
+      const [, configKey] = key.split('test.configuration.')
+      acc[configKey] = tags[key]
+    }
+    return acc
+  }, {})
+}
+
 function getIsTestSessionTrace (trace) {
   return trace.some(span =>
     span.type === 'test_session_end' || span.type === 'test_suite_end' || span.type === 'test_module_end'
@@ -95,6 +108,7 @@ class CiVisibilityExporter extends AgentInfoExporter {
         env: this._config.env,
         service: this._config.service,
         isEvpProxy: !!this._isUsingEvpProxy,
+        custom: getTestConfigurationTags(this._config.tags),
         ...testConfiguration
       }
       getSkippableSuitesRequest(configuration, callback)
@@ -119,6 +133,7 @@ class CiVisibilityExporter extends AgentInfoExporter {
         env: this._config.env,
         service: this._config.service,
         isEvpProxy: !!this._isUsingEvpProxy,
+        custom: getTestConfigurationTags(this._config.tags),
         ...testConfiguration
       }
       getItrConfigurationRequest(configuration, (err, itrConfig) => {

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -10,7 +10,9 @@ const {
   getLatestCommits,
   getRepositoryUrl,
   generatePackFilesForCommits,
-  getCommitsToUpload
+  getCommitsToUpload,
+  isShallowRepository,
+  unshallowRepository
 } = require('../../../plugins/util/git')
 
 const isValidSha = (sha) => /[0-9a-f]{40}/.test(sha)
@@ -157,6 +159,10 @@ function sendGitMetadata (url, isEvpProxy, callback) {
     return callback(new Error('Repository URL is empty'))
   }
 
+  if (isShallowRepository()) {
+    unshallowRepository()
+  }
+
   getCommitsToExclude({ url, repositoryUrl, isEvpProxy }, (err, commitsToExclude, headCommit) => {
     if (err) {
       return callback(err)

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js

@@ -13,7 +13,8 @@ function getItrConfiguration ({
   osArchitecture,
   runtimeName,
   runtimeVersion,
-  branch
+  branch,
+  custom
 }, done) {
   const options = {
     path: '/api/v2/libraries/tests/services/setting',
@@ -53,7 +54,8 @@ function getItrConfiguration ({
           'os.version': osVersion,
           'os.architecture': osArchitecture,
           'runtime.name': runtimeName,
-          'runtime.version': runtimeVersion
+          'runtime.version': runtimeVersion,
+          custom
         },
         service,
         env,

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js

@@ -11,7 +11,8 @@ function getSkippableSuites ({
   osPlatform,
   osArchitecture,
   runtimeName,
-  runtimeVersion
+  runtimeVersion,
+  custom
 }, done) {
   const options = {
     path: '/api/v2/ci/tests/skippable',
@@ -51,7 +52,8 @@ function getSkippableSuites ({
           'os.version': osVersion,
           'os.architecture': osArchitecture,
           'runtime.name': runtimeName,
-          'runtime.version': runtimeVersion
+          'runtime.version': runtimeVersion,
+          custom
         },
         service,
         env,

package/packages/dd-trace/src/config.js

@@ -376,7 +376,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
 
     const DD_CIVISIBILITY_GIT_UPLOAD_ENABLED = coalesce(
       process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED,
-      false
+      true
     )
 
     const ingestion = options.ingestion || {}

package/packages/dd-trace/src/lambda/handler.js

@@ -22,15 +22,14 @@ let __lambdaTimeout
  * @param {*} context AWS Lambda context object.
  */
 function checkTimeout (context) {
-  let remainingTimeInMillis = context.getRemainingTimeInMillis()
-  const apmFlushDeadline = parseInt(process.env.DD_APM_FLUSH_DEADLINE)
-  if (apmFlushDeadline && apmFlushDeadline <= remainingTimeInMillis) {
-    remainingTimeInMillis = apmFlushDeadline
-  }
+  const remainingTimeInMillis = context.getRemainingTimeInMillis()
+
+  let apmFlushDeadline = parseInt(process.env.DD_APM_FLUSH_DEADLINE_MILLISECONDS) || 100
+  apmFlushDeadline = apmFlushDeadline < 0 ? 100 : apmFlushDeadline
 
   __lambdaTimeout = setTimeout(() => {
     timeoutChannel.publish(undefined)
-  }, remainingTimeInMillis - 50)
+  }, remainingTimeInMillis - apmFlushDeadline)
 }
 
 /**

package/packages/dd-trace/src/log/writer.js

@@ -78,39 +78,47 @@ function reset () {
 }
 
 function onError (err) {
-  if (enabled) {
-    if (typeof err !== 'object' || !err) {
-      err = String(err)
-    } else if (!err.stack) {
-      err = String(err.message || err)
-    }
-
-    if (typeof err === 'string') {
-      err = new Error(err)
-    }
-
-    withNoop(() => logger.error(err))
-  }
+  if (enabled) error(err)
 }
 
 function onWarn (message) {
-  if (!logger.warn) return onDebug(message)
-  if (enabled) {
-    withNoop(() => logger.warn(message))
-  }
+  if (enabled) warn(message)
 }
 
 function onInfo (message) {
-  if (!logger.info) return onDebug(message)
-  if (enabled) {
-    withNoop(() => logger.info(message))
-  }
+  if (enabled) info(message)
 }
 
 function onDebug (message) {
-  if (enabled) {
-    withNoop(() => logger.debug(message))
+  if (enabled) debug(message)
+}
+
+function error (err) {
+  if (typeof err !== 'object' || !err) {
+    err = String(err)
+  } else if (!err.stack) {
+    err = String(err.message || err)
   }
+
+  if (typeof err === 'string') {
+    err = new Error(err)
+  }
+
+  withNoop(() => logger.error(err))
+}
+
+function warn (message) {
+  if (!logger.warn) return debug(message)
+  withNoop(() => logger.warn(message))
+}
+
+function info (message) {
+  if (!logger.info) return debug(message)
+  withNoop(() => logger.info(message))
+}
+
+function debug (message) {
+  withNoop(() => logger.debug(message))
 }
 
-module.exports = { use, toggle, reset }
+module.exports = { use, toggle, reset, error, warn, info, debug }