dd-trace 4.13.1 → 4.14.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "4.13.1",
+  "version": "4.14.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -68,7 +68,7 @@
   },
   "dependencies": {
     "@datadog/native-appsec": "^3.2.0",
-    "@datadog/native-iast-rewriter": "2.0.1",
+    "@datadog/native-iast-rewriter": "2.1.3",
     "@datadog/native-iast-taint-tracking": "1.5.0",
     "@datadog/native-metrics": "^2.0.0",
     "@datadog/pprof": "3.2.0",

package/packages/datadog-instrumentations/src/mocha.js

@@ -441,6 +441,9 @@ addHook({
   file: 'lib/cli/run-helpers.js'
 }, (run) => {
   shimmer.wrap(run, 'runMocha', runMocha => async function () {
+    if (!testStartCh.hasSubscribers) {
+      return runMocha.apply(this, arguments)
+    }
     const mocha = arguments[0]
     /**
      * This attaches `run` to the global context, which we'll call after

package/packages/datadog-plugin-cypress/src/plugin.js

@@ -25,6 +25,7 @@ const {
 } = require('../../dd-trace/src/plugins/util/test')
 const { ORIGIN_KEY, COMPONENT } = require('../../dd-trace/src/constants')
 const log = require('../../dd-trace/src/log')
+const NoopTracer = require('../../dd-trace/src/noop/tracer')
 
 const TEST_FRAMEWORK_NAME = 'cypress'
 
@@ -119,10 +120,32 @@ function getSkippableTests (isSuitesSkippingEnabled, tracer, testConfiguration)
   })
 }
 
+const noopTask = {
+  'dd:testSuiteStart': () => {
+    return null
+  },
+  'dd:beforeEach': () => {
+    return {}
+  },
+  'dd:afterEach': () => {
+    return null
+  },
+  'dd:addTags': () => {
+    return null
+  }
+}
+
 module.exports = (on, config) => {
   let isTestsSkipped = false
   const skippedTests = []
   const tracer = require('../../dd-trace')
+
+  // The tracer was not init correctly for whatever reason (such as invalid DD_SITE)
+  if (tracer._tracer instanceof NoopTracer) {
+    // We still need to register these tasks or the support file will fail
+    return on('task', noopTask)
+  }
+
   const testEnvironmentMetadata = getTestEnvironmentMetadata(TEST_FRAMEWORK_NAME)
 
   const {
@@ -328,12 +351,16 @@ module.exports = (on, config) => {
     }
 
     return new Promise(resolve => {
-      if (tracer._tracer._exporter.flush) {
-        tracer._tracer._exporter.flush(() => {
+      const exporter = tracer._tracer._exporter
+      if (!exporter) {
+        return resolve(null)
+      }
+      if (exporter.flush) {
+        exporter.flush(() => {
           resolve(null)
         })
-      } else {
-        tracer._tracer._exporter._writer.flush(() => {
+      } else if (exporter._writer) {
+        exporter._writer.flush(() => {
           resolve(null)
         })
       }
@@ -375,7 +402,7 @@ module.exports = (on, config) => {
     'dd:afterEach': ({ test, coverage }) => {
       const { state, error, isRUMActive, testSourceLine, testSuite, testName } = test
       if (activeSpan) {
-        if (coverage && tracer._tracer._exporter.exportCoverage && isCodeCoverageEnabled) {
+        if (coverage && isCodeCoverageEnabled && tracer._tracer._exporter && tracer._tracer._exporter.exportCoverage) {
           const coverageFiles = getCoveredFilenamesFromCoverage(coverage)
           const relativeCoverageFiles = coverageFiles.map(file => getTestSuitePath(file, rootDir))
           const { _traceId, _spanId } = testSuiteSpan.context()

package/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js

@@ -6,6 +6,7 @@ const { addVulnerability } = require('../vulnerability-reporter')
 const { getIastContext } = require('../iast-context')
 const overheadController = require('../overhead-controller')
 const { SinkIastPlugin } = require('../iast-plugin')
+const { getOriginalPathAndLineFromSourceMap } = require('../taint-tracking/rewriter')
 
 class Analyzer extends SinkIastPlugin {
   constructor (type) {
@@ -25,8 +26,9 @@ class Analyzer extends SinkIastPlugin {
     const evidence = this._getEvidence(value, context)
     const location = this._getLocation(value)
     if (!this._isExcluded(location)) {
+      const locationSourceMap = this._replaceLocationFromSourceMap(location)
       const spanId = context && context.rootSpan && context.rootSpan.context().toSpanId()
-      const vulnerability = this._createVulnerability(this._type, evidence, spanId, location)
+      const vulnerability = this._createVulnerability(this._type, evidence, spanId, locationSourceMap)
       addVulnerability(context, vulnerability)
     }
   }
@@ -47,6 +49,22 @@ class Analyzer extends SinkIastPlugin {
     return getFirstNonDDPathAndLine(this._getExcludedPaths())
   }
 
+  _replaceLocationFromSourceMap (location) {
+    if (location) {
+      const { path, line, column } = getOriginalPathAndLineFromSourceMap(location)
+      if (path) {
+        location.path = path
+      }
+      if (line) {
+        location.line = line
+      }
+      if (column) {
+        location.column = column
+      }
+    }
+    return location
+  }
+
   _getExcludedPaths () {}
 
   _isInvalidContext (store, iastContext) {

package/packages/dd-trace/src/appsec/iast/path-line.js

@@ -47,6 +47,7 @@ function getFirstNonDDPathAndLineFromCallsites (callsites, externallyExcludedPat
         return {
           path: path.relative(process.cwd(), filepath),
           line: callsite.getLineNumber(),
+          column: callsite.getColumnNumber(),
           isInternal: !path.isAbsolute(filepath)
         }
       }

package/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter.js

@@ -10,12 +10,51 @@ const { getRewriteFunction } = require('./rewriter-telemetry')
 
 let rewriter
 let getPrepareStackTrace
+
+let getRewriterOriginalPathAndLineFromSourceMap = function (path, line, column) {
+  return { path, line, column }
+}
+
+function isFlagPresent (flag) {
+  return process.env.NODE_OPTIONS?.includes(flag) ||
+    process.execArgv?.some(arg => arg.includes(flag))
+}
+
+function getGetOriginalPathAndLineFromSourceMapFunction (chainSourceMap, getOriginalPathAndLineFromSourceMap) {
+  if (chainSourceMap) {
+    return function (path, line, column) {
+      // if --enable-source-maps is present stacktraces of the rewritten files contain the original path, file and
+      // column because the sourcemap chaining is done during the rewriting process so we can skip it
+      if (isPrivateModule(path) && isNotLibraryFile(path)) {
+        return { path, line, column }
+      } else {
+        return getOriginalPathAndLineFromSourceMap(path, line, column)
+      }
+    }
+  } else {
+    return getOriginalPathAndLineFromSourceMap
+  }
+}
+
 function getRewriter (telemetryVerbosity) {
   if (!rewriter) {
-    const iastRewriter = require('@datadog/native-iast-rewriter')
-    const Rewriter = iastRewriter.Rewriter
-    getPrepareStackTrace = iastRewriter.getPrepareStackTrace
-    rewriter = new Rewriter({ csiMethods, telemetryVerbosity: getName(telemetryVerbosity) })
+    try {
+      const iastRewriter = require('@datadog/native-iast-rewriter')
+      const Rewriter = iastRewriter.Rewriter
+      getPrepareStackTrace = iastRewriter.getPrepareStackTrace
+
+      const chainSourceMap = isFlagPresent('--enable-source-maps')
+      const getOriginalPathAndLineFromSourceMap = iastRewriter.getOriginalPathAndLineFromSourceMap
+      if (getOriginalPathAndLineFromSourceMap) {
+        getRewriterOriginalPathAndLineFromSourceMap =
+          getGetOriginalPathAndLineFromSourceMapFunction(chainSourceMap, getOriginalPathAndLineFromSourceMap)
+      }
+
+      rewriter = new Rewriter({ csiMethods, telemetryVerbosity: getName(telemetryVerbosity), chainSourceMap })
+    } catch (e) {
+      iastLog.error('Unable to initialize TaintTracking Rewriter')
+        .errorAndPublish(e)
+    }
   }
   return rewriter
 }
@@ -74,6 +113,10 @@ function disableRewriter () {
   Error.prepareStackTrace = originalPrepareStackTrace
 }
 
+function getOriginalPathAndLineFromSourceMap ({ path, line, column }) {
+  return getRewriterOriginalPathAndLineFromSourceMap(path, line, column)
+}
+
 module.exports = {
-  enableRewriter, disableRewriter
+  enableRewriter, disableRewriter, getOriginalPathAndLineFromSourceMap
 }

package/packages/dd-trace/src/appsec/recommended.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.7.1"
+    "rules_version": "1.7.2"
   },
   "rules": [
     {
@@ -1743,7 +1743,10 @@
               "sys/hypervisor",
               "sys/kernel",
               "sys/module",
-              "sys/power"
+              "sys/power",
+              "windows\\win.ini",
+              "default\\ntuser.dat",
+              "/var/run/secrets/kubernetes.io/serviceaccount"
             ]
           },
           "operator": "phrase_match"
@@ -2312,7 +2315,8 @@
         }
       ],
       "transformers": [
-        "lowercase"
+        "lowercase",
+        "cmdLine"
       ]
     },
     {
@@ -2950,7 +2954,7 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "[\\s\\\"'`;\\/0-9=\\x0B\\x09\\x0C\\x3B\\x2C\\x28\\x3B]on(?:d(?:r(?:ag(?:en(?:ter|d)|leave|start|over)?|op)|urationchange|blclick)|s(?:e(?:ek(?:ing|ed)|arch|lect)|u(?:spend|bmit)|talled|croll|how)|m(?:ouse(?:(?:lea|mo)ve|o(?:ver|ut)|enter|down|up)|essage)|p(?:a(?:ge(?:hide|show)|(?:st|us)e)|lay(?:ing)?|rogress)|c(?:anplay(?:through)?|o(?:ntextmenu|py)|hange|lick|ut)|a(?:nimation(?:iteration|start|end)|(?:fterprin|bor)t)|t(?:o(?:uch(?:cancel|start|move|end)|ggle)|imeupdate)|f(?:ullscreen(?:change|error)|ocus(?:out|in)?)|(?:(?:volume|hash)chang|o(?:ff|n)lin)e|b(?:efore(?:unload|print)|lur)|load(?:ed(?:meta)?data|start)?|r(?:es(?:ize|et)|atechange)|key(?:press|down|up)|w(?:aiting|heel)|in(?:valid|put)|e(?:nded|rror)|unload)[\\s\\x0B\\x09\\x0C\\x3B\\x2C\\x28\\x3B]*?=[^=]",
+            "regex": "\\bon(?:d(?:r(?:ag(?:en(?:ter|d)|leave|start|over)?|op)|urationchange|blclick)|s(?:e(?:ek(?:ing|ed)|arch|lect)|u(?:spend|bmit)|talled|croll|how)|m(?:ouse(?:(?:lea|mo)ve|o(?:ver|ut)|enter|down|up)|essage)|p(?:a(?:ge(?:hide|show)|(?:st|us)e)|lay(?:ing)?|rogress|aste|ointer(?:cancel|down|enter|leave|move|out|over|rawupdate|up))|c(?:anplay(?:through)?|o(?:ntextmenu|py)|hange|lick|ut)|a(?:nimation(?:iteration|start|end)|(?:fterprin|bor)t|uxclick|fterscriptexecute)|t(?:o(?:uch(?:cancel|start|move|end)|ggle)|imeupdate)|f(?:ullscreen(?:change|error)|ocus(?:out|in)?|inish)|(?:(?:volume|hash)chang|o(?:ff|n)lin)e|b(?:efore(?:unload|print)|lur)|load(?:ed(?:meta)?data|start|end)?|r(?:es(?:ize|et)|atechange)|key(?:press|down|up)|w(?:aiting|heel)|in(?:valid|put)|e(?:nded|rror)|unload)[\\s\\x0B\\x09\\x0C\\x3B\\x2C\\x28\\x3B]*?=[^=]",
             "options": {
               "min_length": 8
             }
@@ -4636,6 +4640,46 @@
       ],
       "transformers": []
     },
+    {
+      "id": "dog-913-008",
+      "name": "Netsparker OOB domain",
+      "tags": {
+        "type": "commercial_scanner",
+        "category": "attack_attempt",
+        "tool_name": "Netsparker",
+        "confidence": "0"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              },
+              {
+                "address": "server.request.headers.no_cookies"
+              },
+              {
+                "address": "grpc.server.request.message"
+              }
+            ],
+            "regex": "\\b(?:\\.|(?:\\\\|&#)(?:0*46|x0*2e);)r87(?:\\.|(?:\\\\|&#)(?:0*46|x0*2e);)(?:me|com)\\b",
+            "options": {
+              "case_sensitive": false,
+              "min_length": 7
+            }
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": []
+    },
     {
       "id": "dog-931-001",
       "name": "RFI: URL Payload to well known RFI target",
@@ -4699,6 +4743,56 @@
       ],
       "transformers": []
     },
+    {
+      "id": "dog-941-001",
+      "name": "XSS in source property",
+      "tags": {
+        "type": "xss",
+        "category": "attack_attempt",
+        "confidence": "0"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "user-agent"
+                ]
+              },
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "referer"
+                ]
+              },
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              },
+              {
+                "address": "grpc.server.request.message"
+              }
+            ],
+            "regex": "<(?:iframe|esi:include)(?:(?:\\s|/)*\\w+=[\"'\\w]+)*(?:\\s|/)*src(?:doc)?=[\"']?(?:data:|javascript:|http:|//)[^\\s'\"]+['\"]?",
+            "options": {
+              "min_length": 14
+            }
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": [
+        "removeNulls",
+        "urlDecodeUni"
+      ]
+    },
     {
       "id": "dog-942-001",
       "name": "Blind XSS callback domains",
@@ -5428,12 +5522,14 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "(?i)[&|]\\s*cat\\s+\\/etc\\/[\\w\\.\\/]*passwd\\s*[&|]"
+            "regex": "(?i)[&|]\\s*cat\\s*\\/etc\\/[\\w\\.\\/]*passwd\\s*[&|]"
           },
           "operator": "match_regex"
         }
       ],
-      "transformers": []
+      "transformers": [
+        "cmdLine"
+      ]
     },
     {
       "id": "sqr-000-010",
@@ -7014,7 +7110,10 @@
                 ]
               }
             ],
-            "regex": "mozilla/4\\.0 \\(compatible(; msie 6\\.0; win32)?\\)"
+            "regex": "mozilla/4\\.0 \\(compatible(; msie (?:6\\.0; win32|4\\.0; Windows NT))?\\)",
+            "options": {
+              "case_sensitive": false
+            }
           },
           "operator": "match_regex"
         }
@@ -7076,4 +7175,4 @@
       "transformers": []
     }
   ]
-}
+}

package/packages/dd-trace/src/dogstatsd.js

@@ -5,6 +5,7 @@ const request = require('./exporters/common/request')
 const dgram = require('dgram')
 const isIP = require('net').isIP
 const log = require('./log')
+const { URL, format } = require('url')
 
 const MAX_BUFFER_SIZE = 1024 // limit from the agent
 
@@ -13,9 +14,7 @@ const TYPE_GAUGE = 'g'
 const TYPE_DISTRIBUTION = 'd'
 
 class DogStatsDClient {
-  constructor (options) {
-    options = options || {}
-
+  constructor (options = {}) {
     if (options.metricsProxyUrl) {
       this._httpOptions = {
         url: options.metricsProxyUrl.toString(),
@@ -50,6 +49,8 @@ class DogStatsDClient {
   flush () {
     const queue = this._enqueue()
 
+    log.debug(`Flushing ${queue.length} metrics via ${this._httpOptions ? 'HTTP' : 'UDP'}`)
+
     if (this._queue.length === 0) return
 
     this._queue = []
@@ -141,6 +142,44 @@ class DogStatsDClient {
 
     return socket
   }
+
+  static generateClientConfig (config = {}) {
+    const tags = []
+
+    if (config.tags) {
+      Object.keys(config.tags)
+        .filter(key => typeof config.tags[key] === 'string')
+        .filter(key => {
+          // Skip runtime-id unless enabled as cardinality may be too high
+          if (key !== 'runtime-id') return true
+          return (config.experimental && config.experimental.runtimeId)
+        })
+        .forEach(key => {
+          // https://docs.datadoghq.com/tagging/#defining-tags
+          const value = config.tags[key].replace(/[^a-z0-9_:./-]/ig, '_')
+
+          tags.push(`${key}:${value}`)
+        })
+    }
+
+    const clientConfig = {
+      host: config.dogstatsd.hostname,
+      port: config.dogstatsd.port,
+      tags
+    }
+
+    if (config.url) {
+      clientConfig.metricsProxyUrl = config.url
+    } else if (config.port) {
+      clientConfig.metricsProxyUrl = new URL(format({
+        protocol: 'http:',
+        hostname: config.hostname || 'localhost',
+        port: config.port
+      }))
+    }
+
+    return clientConfig
+  }
 }
 
 class NoopDogStatsDClient {
@@ -155,8 +194,9 @@ class NoopDogStatsDClient {
 
 // This is a simplified user-facing proxy to the underlying DogStatsDClient instance
 class CustomMetrics {
-  constructor (options) {
-    this.dogstatsd = new DogStatsDClient(options)
+  constructor (config) {
+    const clientConfig = DogStatsDClient.generateClientConfig(config)
+    this.dogstatsd = new DogStatsDClient(clientConfig)
   }
 
   increment (stat, value = 1, tags) {

package/packages/dd-trace/src/plugins/util/test.js

@@ -139,13 +139,13 @@ function removeInvalidMetadata (metadata) {
   return Object.keys(metadata).reduce((filteredTags, tag) => {
     if (tag === GIT_REPOSITORY_URL) {
       if (!validateGitRepositoryUrl(metadata[GIT_REPOSITORY_URL])) {
-        log.error('DD_GIT_REPOSITORY_URL must be a valid URL')
+        log.error(`Repository URL is not a valid repository URL: ${metadata[GIT_REPOSITORY_URL]}.`)
         return filteredTags
       }
     }
     if (tag === GIT_COMMIT_SHA) {
       if (!validateGitCommitSha(metadata[GIT_COMMIT_SHA])) {
-        log.error('DD_GIT_COMMIT_SHA must be a full-length git SHA')
+        log.error(`Git commit SHA must be a full-length git SHA: ${metadata[GIT_COMMIT_SHA]}.`)
         return filteredTags
       }
     }

package/packages/dd-trace/src/profiling/profilers/wall.js

@@ -6,9 +6,11 @@ const dc = require('../../../../diagnostics_channel')
 const { HTTP_METHOD, HTTP_ROUTE, RESOURCE_NAME, SPAN_TYPE } = require('../../../../../ext/tags')
 const { WEB } = require('../../../../../ext/types')
 const runtimeMetrics = require('../../runtime_metrics')
+const telemetryMetrics = require('../../telemetry/metrics')
 
 const beforeCh = dc.channel('dd-trace:storage:before')
 const enterCh = dc.channel('dd-trace:storage:enter')
+const profilerTelemetryMetrics = telemetryMetrics.manager.namespace('profilers')
 
 let kSampleCount
 
@@ -168,6 +170,16 @@ class NativeWallProfiler {
     }
   }
 
+  _reportV8bug (maybeBug) {
+    const tag = `v8_profiler_bug_workaround_enabled:${this._v8ProfilerBugWorkaroundEnabled}`
+    const metric = `v8_cpu_profiler${maybeBug ? '_maybe' : ''}_stuck_event_loop`
+    this._logger?.warn(`Wall profiler: ${maybeBug ? 'possible ' : ''}v8 profiler stuck event loop detected.`)
+    // report as runtime metric (can be removed in the future when telemetry is mature)
+    runtimeMetrics.increment(`runtime.node.profiler.${metric}`, tag, true)
+    // report as telemetry metric
+    profilerTelemetryMetrics.count(metric, [tag]).inc()
+  }
+
   _stop (restart) {
     if (!this._started) return
     if (this._codeHotspotsEnabled) {
@@ -178,12 +190,8 @@ class NativeWallProfiler {
     const profile = this._pprof.time.stop(restart, this._codeHotspotsEnabled ? generateLabels : undefined)
     if (restart) {
       const v8BugDetected = this._pprof.time.v8ProfilerStuckEventLoopDetected()
-      if (v8BugDetected === 1) {
-        this._logger?.warn('Wall profiler: possible v8 profiler stuck event loop detected.')
-        runtimeMetrics.increment('runtime.node.profiler.v8_cpu_profiler_maybe_stuck_event_loop', undefined, true)
-      } else if (v8BugDetected === 2) {
-        this._logger?.warn('Wall profiler: v8 profiler stuck event loop detected.')
-        runtimeMetrics.increment('runtime.node.profiler.v8_cpu_profiler_stuck_event_loop', undefined, true)
+      if (v8BugDetected !== 0) {
+        this._reportV8bug(v8BugDetected === 1)
       }
     }
     return profile

package/packages/dd-trace/src/proxy.js

@@ -30,17 +30,7 @@ class Tracer extends NoopProxy {
 
       if (config.dogstatsd) {
         // Custom Metrics
-        this.dogstatsd = new dogstatsd.CustomMetrics({
-          host: config.dogstatsd.hostname,
-          port: config.dogstatsd.port,
-          tags: [
-            // these are the Runtime Metrics default tags
-            // Python also uses these as default Custom Metrics tags
-            `service:${config.tags.service}`,
-            `env:${config.tags.env}`,
-            `version:${config.tags.version}`
-          ]
-        })
+        this.dogstatsd = new dogstatsd.CustomMetrics(config)
 
         setInterval(() => {
           this.dogstatsd.flush()

package/packages/dd-trace/src/runtime_metrics.js

@@ -2,7 +2,6 @@
 
 // TODO: capture every second and flush every 10 seconds
 
-const { URL, format } = require('url')
 const v8 = require('v8')
 const os = require('os')
 const { DogStatsDClient } = require('./dogstatsd')
@@ -27,21 +26,7 @@ reset()
 
 module.exports = {
   start (config) {
-    const tags = []
-
-    Object.keys(config.tags)
-      .filter(key => typeof config.tags[key] === 'string')
-      .filter(key => {
-        // Skip runtime-id unless enabled as cardinality may be too high
-        if (key !== 'runtime-id') return true
-        return (config.experimental && config.experimental.runtimeId)
-      })
-      .forEach(key => {
-        // https://docs.datadoghq.com/tagging/#defining-tags
-        const value = config.tags[key].replace(/[^a-z0-9_:./-]/ig, '_')
-
-        tags.push(`${key}:${value}`)
-      })
+    const clientConfig = DogStatsDClient.generateClientConfig(config)
 
     try {
       nativeMetrics = require('@datadog/native-metrics')
@@ -51,22 +36,6 @@ module.exports = {
       nativeMetrics = null
     }
 
-    const clientConfig = {
-      host: config.dogstatsd.hostname,
-      port: config.dogstatsd.port,
-      tags
-    }
-
-    if (config.url) {
-      clientConfig.metricsProxyUrl = config.url
-    } else if (config.port) {
-      clientConfig.metricsProxyUrl = new URL(format({
-        protocol: 'http:',
-        hostname: config.hostname || 'localhost',
-        port: config.port
-      }))
-    }
-
     client = new DogStatsDClient(clientConfig)
 
     time = process.hrtime()