dd-trace 3.16.0 → 3.17.1

package/LICENSE-3rdparty.csv

@@ -36,6 +36,7 @@ dev,chalk,MIT,Copyright Sindre Sorhus
 dev,checksum,MIT,Copyright Daniel D. Shaw
 dev,cli-table3,MIT,Copyright 2014 James Talmage
 dev,dotenv,BSD-2-Clause,Copyright 2015 Scott Motte
+dev,esbuild,MIT,Copyright (c) 2020 Evan Wallace
 dev,eslint,MIT,Copyright JS Foundation and other contributors https://js.foundation
 dev,eslint-config-standard,MIT,Copyright Feross Aboukhadijeh
 dev,eslint-plugin-import,MIT,Copyright 2015 Ben Mosher

package/README.md

@@ -151,6 +151,19 @@ $ yarn lint
 ```
 
 
+### Experimental ESM Support
+
+ESM support is currently in the experimental stages, while CJS has been supported
+since inception. This means that code loaded using `require()` should work fine
+but code loaded using `import` might not always work.
+
+Use the following command to enable experimental ESM support with your application:
+
+```sh
+node --loader dd-trace/loader-hook.mjs entrypoint.js
+```
+
+
 ### Benchmarks
 
 Our microbenchmarks live in `benchmark/sirun`. Each directory in there
@@ -175,6 +188,42 @@ That said, even if your application runs on Lambda, any core instrumentation iss
 Regardless of where you open the issue, someone at Datadog will try to help.
 
 
+## Bundling
+
+Generally, `dd-trace` works by intercepting `require()` calls that a Node.js application makes when loading modules. This includes modules that are built-in to Node.js, like the `fs` module for accessing the filesystem, as well as modules installed from the npm registry, like the `pg` database module.
+
+Also generally, bundlers work by crawling all of the `require()` calls that an application makes to files on disk, replacing the `require()` calls with custom code, and then concatenating all of the resulting JavaScript into one "bundled" file. When a built-in module is loaded, like `require('fs')`, that call can then remain the same in the resulting bundle.
+
+Fundamentally APM tools like `dd-trace` stop working at this point. Perhaps they continue to intercept the calls for built-in modules but don't intercept calls to third party libraries. This means that by default when you bundle a `dd-trace` app with a bundler it is likely to capture information about disk access (via `fs`) and outbound HTTP requests (via `http`), but will otherwise omit calls to third party libraries (like extracting incoming request route information for the `express` framework or showing which query is run for the `mysql` database client).
+
+To get around this, one can treat all third party modules, or at least third party modules that the APM needs to instrument, as being "external" to the bundler. With this setting the instrumented modules remain on disk and continue to be loaded via `require()` while the non-instrumented modules are bundled. Sadly this results in a build with many extraneous files and starts to defeat the purpose of bundling. 
+
+For these reasons it's necessary to have custom-built bundler plugins. Such plugins are able to instruct the bundler on how to behave, injecting intermediary code and otherwise intercepting the "translated" `require()` calls. The result is that many more packages are then included in the bundled JavaScript file. Some applications can have 100% of modules bundled, however native modules still need to remain external to the bundle.
+
+### Esbuild Support
+
+This library provides experimental esbuild support in the form of an esbuild plugin, and currently requires at least Node.js v16.17 or v18.7. To use the plugin, make sure you have `dd-trace@3+` installed, and then require the `dd-trace/esbuild` module when building your bundle.
+
+Here's an example of how one might use `dd-trace` with esbuild:
+
+```javascript
+const ddPlugin = require('dd-trace/esbuild')
+const esbuild = require('esbuild')
+
+esbuild.build({
+  entryPoints: ['app.js'], 
+  bundle: true,
+  outfile: 'out.js',
+  plugins: [ddPlugin],
+  platform: 'node', // allows built-in modules to be required
+  target: ['node16']
+}).catch((err) => {
+  console.error(err)
+  process.exit(1)
+})
+```
+
+
 ## Security Vulnerabilities
 
 If you have found a security issue, please contact the security team directly at [security@datadoghq.com](mailto:security@datadoghq.com).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "3.16.0",
+  "version": "3.17.1",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -32,7 +32,10 @@
     "test:plugins:upstream": "node ./packages/dd-trace/test/plugins/suite.js",
     "test:profiler": "tap \"packages/dd-trace/test/profiling/**/*.spec.js\"",
     "test:profiler:ci": "npm run test:profiler -- --coverage --nyc-arg=--include=\"packages/dd-trace/src/profiling/**/*.js\"",
-    "test:integration": "mocha --colors --timeout 30000 \"integration-tests/**/*.spec.js\"",
+    "test:integration": "mocha --colors --timeout 30000 \"integration-tests/*.spec.js\"",
+    "test:integration:cucumber": "mocha --colors --timeout 30000 \"integration-tests/cucumber/*.spec.js\"",
+    "test:integration:cypress": "mocha --colors --timeout 30000 \"integration-tests/cypress/*.spec.js\"",
+    "test:integration:playwright": "mocha --colors --timeout 30000 \"integration-tests/playwright/*.spec.js\"",
     "test:shimmer": "mocha --colors 'packages/datadog-shimmer/test/**/*.spec.js'",
     "test:shimmer:ci": "nyc --no-clean --include 'packages/datadog-shimmer/src/**/*.js' -- npm run test:shimmer",
     "leak:core": "node ./scripts/install_plugin_modules && (cd packages/memwatch && yarn) && NODE_PATH=./packages/memwatch/node_modules node --no-warnings ./node_modules/.bin/tape 'packages/dd-trace/test/leak/**/*.js'",
@@ -64,7 +67,7 @@
   "dependencies": {
     "@datadog/native-appsec": "2.0.0",
     "@datadog/native-iast-rewriter": "2.0.1",
-    "@datadog/native-iast-taint-tracking": "1.1.1",
+    "@datadog/native-iast-taint-tracking": "1.3.1",
     "@datadog/native-metrics": "^1.5.0",
     "@datadog/pprof": "^2.1.0",
     "@datadog/sketches-js": "^2.1.0",
@@ -88,7 +91,7 @@
     "path-to-regexp": "^0.1.2",
     "protobufjs": "^7.1.2",
     "retry": "^0.10.1",
-    "semver": "^5.5.0"
+    "semver": "^7.3.8"
   },
   "devDependencies": {
     "@types/node": ">=14",
@@ -101,6 +104,7 @@
     "checksum": "^0.1.1",
     "cli-table3": "^0.5.1",
     "dotenv": "8.2.0",
+    "esbuild": "0.16.12",
     "eslint": "^8.23.0",
     "eslint-config-standard": "^11.0.0-beta.0",
     "eslint-plugin-import": "^2.8.0",

package/packages/datadog-esbuild/index.js

@@ -0,0 +1,104 @@
+'use strict'
+
+/* eslint-disable no-console */
+
+const NAMESPACE = 'datadog'
+
+const instrumented = Object.keys(require('../datadog-instrumentations/src/helpers/hooks.js'))
+const rawBuiltins = require('module').builtinModules
+
+warnIfUnsupported()
+
+const builtins = new Set()
+
+for (const builtin of rawBuiltins) {
+  builtins.add(builtin)
+  builtins.add(`node:${builtin}`)
+}
+
+const packagesOfInterest = new Set()
+
+const DEBUG = !!process.env.DD_TRACE_DEBUG
+
+// We don't want to handle any built-in packages via DCITM
+// Those packages will still be handled via RITM
+// Attempting to instrument them would fail as they have no package.json file
+for (const pkg of instrumented) {
+  if (builtins.has(pkg)) continue
+  if (pkg.startsWith('node:')) continue
+  packagesOfInterest.add(pkg)
+}
+
+const DC_CHANNEL = 'dd-trace:bundledModuleLoadStart'
+
+module.exports.name = 'datadog-esbuild'
+
+module.exports.setup = function (build) {
+  build.onResolve({ filter: /.*/ }, args => {
+    const packageName = args.path
+
+    if (args.namespace === 'file' && packagesOfInterest.has(packageName)) {
+      // The file namespace is used when requiring files from disk in userland
+      const pathToPackageJson = require.resolve(`${packageName}/package.json`, { paths: [ args.resolveDir ] })
+      const pkg = require(pathToPackageJson)
+
+      if (DEBUG) {
+        console.log(`resolve ${packageName}@${pkg.version}`)
+      }
+
+      // https://esbuild.github.io/plugins/#on-resolve-arguments
+      return {
+        path: packageName,
+        namespace: NAMESPACE,
+        pluginData: {
+          version: pkg.version
+        }
+      }
+    } else if (args.namespace === 'datadog') {
+      // The datadog namespace is used when requiring files that are injected during the onLoad stage
+      // see note in onLoad
+
+      if (builtins.has(packageName)) return
+
+      return {
+        path: require.resolve(packageName, { paths: [ args.resolveDir ] }),
+        namespace: 'file'
+      }
+    }
+  })
+
+  build.onLoad({ filter: /.*/, namespace: NAMESPACE }, args => {
+    if (DEBUG) {
+      console.log(`load ${args.path}@${args.pluginData.version}`)
+    }
+
+    // JSON.stringify adds double quotes. For perf gain could simply add in quotes when we know it's safe.
+    const contents = `
+      const dc = require('diagnostics_channel');
+      const ch = dc.channel(${JSON.stringify(DC_CHANNEL + ':' + args.path)});
+      const mod = require(${JSON.stringify(args.path)});
+      const payload = {
+        module: mod,
+        path: ${JSON.stringify(args.path)},
+        version: ${JSON.stringify(args.pluginData.version)}
+      };
+      ch.publish(payload);
+      module.exports = payload.module;
+    `
+    // https://esbuild.github.io/plugins/#on-load-results
+    return {
+      contents,
+      loader: 'js'
+    }
+  })
+}
+
+function warnIfUnsupported () {
+  const [major, minor] = process.versions.node.split('.').map(Number)
+  if (major < 14 || (major === 14 && minor < 17)) {
+    console.error('WARNING: Esbuild support isn\'t available for older versions of Node.js.')
+    console.error(`Expected: Node.js >= v14.17. Actual: Node.js = ${process.version}.`)
+    console.error('This application may build properly with this version of Node.js, but unless a')
+    console.error('more recent version is used at runtime, third party packages won\'t be instrumented.')
+  }
+}

package/packages/datadog-instrumentations/src/google-cloud-pubsub.js

@@ -63,7 +63,7 @@ function wrapMethod (method) {
   const api = method.name
 
   return function (request) {
-    if (!requestStartCh.hasSubscribers) return request.apply(this, arguments)
+    if (!requestStartCh.hasSubscribers) return method.apply(this, arguments)
 
     const innerAsyncResource = new AsyncResource('bound-anonymous-fn')
 

package/packages/datadog-instrumentations/src/helpers/hook.js

@@ -3,13 +3,21 @@
 const path = require('path')
 const iitm = require('../../../dd-trace/src/iitm')
 const ritm = require('../../../dd-trace/src/ritm')
-
+const dcitm = require('../../../dd-trace/src/dcitm')
+
+/**
+ * This is called for every module that dd-trace supports instrumentation for.
+ * In practice, `modules` is always an array with a single entry.
+ *
+ * @param {string[]} modules list of modules to hook into
+ * @param {Function} onrequire callback to be executed upon encountering module
+ */
 function Hook (modules, onrequire) {
   if (!(this instanceof Hook)) return new Hook(modules, onrequire)
 
   this._patched = Object.create(null)
 
-  const safeHook = (moduleExports, moduleName, moduleBaseDir) => {
+  const safeHook = (moduleExports, moduleName, moduleBaseDir, moduleVersion) => {
     const parts = [moduleBaseDir, moduleName].filter(v => v)
     const filename = path.join(...parts)
 
@@ -17,7 +25,7 @@ function Hook (modules, onrequire) {
 
     this._patched[filename] = true
 
-    return onrequire(moduleExports, moduleName, moduleBaseDir)
+    return onrequire(moduleExports, moduleName, moduleBaseDir, moduleVersion)
   }
 
   this._ritmHook = ritm(modules, {}, safeHook)
@@ -33,11 +41,13 @@ function Hook (modules, onrequire) {
       return safeHook(moduleExports, moduleName, moduleBaseDir)
     }
   })
+  this._dcitmHook = dcitm(modules, {}, safeHook)
 }
 
 Hook.prototype.unhook = function () {
   this._ritmHook.unhook()
   this._iitmHook.unhook()
+  this._dcitmHook.unhook()
   this._patched = Object.create(null)
 }
 

package/packages/datadog-instrumentations/src/helpers/instrument.js

@@ -14,6 +14,12 @@ exports.channel = function (name) {
   return ch
 }
 
+/**
+ * @param {string} args.name module name
+ * @param {string[]} args.versions array of semver range strings
+ * @param {string} args.file path to file within package to instrument?
+ * @param Function hook
+ */
 exports.addHook = function addHook ({ name, versions, file }, hook) {
   if (!instrumentations[name]) {
     instrumentations[name] = []

package/packages/datadog-instrumentations/src/helpers/register.js

@@ -17,7 +17,7 @@ const loadChannel = channel('dd-trace:instrumentation:load')
 // TODO: make this more efficient
 
 for (const packageName of names) {
-  Hook([packageName], (moduleExports, moduleName, moduleBaseDir) => {
+  Hook([packageName], (moduleExports, moduleName, moduleBaseDir, moduleVersion) => {
     moduleName = moduleName.replace(pathSepExpr, '/')
 
     hooks[packageName]()
@@ -26,7 +26,7 @@ for (const packageName of names) {
       const fullFilename = filename(name, file)
 
       if (moduleName === fullFilename) {
-        const version = getVersion(moduleBaseDir)
+        const version = moduleVersion || getVersion(moduleBaseDir)
 
         if (matchVersion(version, versions)) {
           try {

package/packages/datadog-instrumentations/src/pg.js

@@ -27,27 +27,22 @@ function wrapQuery (query) {
       return query.apply(this, arguments)
     }
 
-    const retval = query.apply(this, arguments)
-
-    const queryQueue = this.queryQueue || this._queryQueue
-    const activeQuery = this.activeQuery || this._activeQuery
-    const pgQuery = queryQueue[queryQueue.length - 1] || activeQuery
-
-    if (!pgQuery) {
-      return retval
-    }
-
     const callbackResource = new AsyncResource('bound-anonymous-fn')
     const asyncResource = new AsyncResource('bound-anonymous-fn')
     const processId = this.processID
+    let pgQuery = {
+      text: arguments[0]
+    }
+
     return asyncResource.runInAsyncScope(() => {
       startCh.publish({
         params: this.connectionParameters,
-        originalQuery: pgQuery.text,
         query: pgQuery,
         processId
       })
 
+      arguments[0] = pgQuery.text
+
       const finish = asyncResource.bind(function (error) {
         if (error) {
           errorCh.publish(error)
@@ -55,6 +50,16 @@ function wrapQuery (query) {
         finishCh.publish()
       })
 
+      const retval = query.apply(this, arguments)
+      const queryQueue = this.queryQueue || this._queryQueue
+      const activeQuery = this.activeQuery || this._activeQuery
+
+      pgQuery = queryQueue[queryQueue.length - 1] || activeQuery
+
+      if (!pgQuery) {
+        return retval
+      }
+
       if (pgQuery.callback) {
         const originalCallback = callbackResource.bind(pgQuery.callback)
         pgQuery.callback = function (err, res) {

package/packages/datadog-plugin-next/src/index.js

@@ -68,6 +68,12 @@ class NextPlugin extends Plugin {
       const span = store.span
       const req = this._requests.get(span)
 
+      // Only use error page names if there's not already a name
+      const current = span.context()._tags['next.page']
+      if (current && (page === '/404' || page === '/500' || page === '/_error')) {
+        return
+      }
+
       span.addTags({
         [COMPONENT]: this.constructor.id,
         'resource.name': `${req.method} ${page}`.trim(),

package/packages/dd-trace/src/appsec/{templates/blocked.html → blocked_templates.js}

@@ -1,4 +1,7 @@
-<!-- Sorry, you’ve been blocked -->
+/* eslint-disable max-len */
+'use strict'
+
+const html = `<!-- Sorry, you've been blocked -->
 <!DOCTYPE html>
 <html lang="en">
 
@@ -97,3 +100,18 @@
 </body>
 
 </html>
+`
+
+const json = `{
+  "errors": [
+    {
+      "title": "You've been blocked",
+      "detail": "Sorry, you cannot access this page. Please contact the customer service team. Security provided by Datadog."
+    }
+  ]
+}`
+
+module.exports = {
+  html,
+  json
+}

package/packages/dd-trace/src/appsec/blocking.js

@@ -1,12 +1,10 @@
 'use strict'
 
 const log = require('../log')
-const fs = require('fs')
+const blockedTemplates = require('./blocked_templates')
 
-// TODO: move template loading to a proper spot.
-let templateLoaded = false
-let templateHtml = ''
-let templateJson = ''
+let templateHtml = blockedTemplates.html
+let templateJson = blockedTemplates.json
 
 function block (req, res, rootSpan, abortController) {
   if (res.headersSent) {
@@ -42,29 +40,16 @@ function block (req, res, rootSpan, abortController) {
   }
 }
 
-function loadTemplates (config) {
-  if (!templateLoaded) {
-    templateHtml = fs.readFileSync(config.appsec.blockedTemplateHtml)
-    templateJson = fs.readFileSync(config.appsec.blockedTemplateJson)
-    templateLoaded = true
+function setTemplates (config) {
+  if (config.appsec.blockedTemplateHtml) {
+    templateHtml = config.appsec.blockedTemplateHtml
   }
-}
-
-async function loadTemplatesAsync (config) {
-  if (!templateLoaded) {
-    templateHtml = await fs.promises.readFile(config.appsec.blockedTemplateHtml)
-    templateJson = await fs.promises.readFile(config.appsec.blockedTemplateJson)
-    templateLoaded = true
+  if (config.appsec.blockedTemplateJson) {
+    templateJson = config.appsec.blockedTemplateJson
   }
 }
 
-function resetTemplates () {
-  templateLoaded = false
-}
-
 module.exports = {
   block,
-  loadTemplates,
-  loadTemplatesAsync,
-  resetTemplates
+  setTemplates
 }

package/packages/dd-trace/src/appsec/iast/analyzers/path-traversal-analyzer.js

@@ -1,4 +1,6 @@
 'use strict'
+
+const path = require('path')
 const { getIastContext } = require('../iast-context')
 const { storage } = require('../../../../../datadog-core')
 const InjectionAnalyzer = require('./injection-analyzer')
@@ -37,6 +39,16 @@ class PathTraversalAnalyzer extends InjectionAnalyzer {
       }
       this.analyze(pathArguments)
     })
+
+    this.exclusionList = [ path.join('node_modules', 'send') + path.sep ]
+  }
+
+  _isExcluded (location) {
+    let ret = false
+    if (location && location.path) {
+      ret = this.exclusionList.some(elem => location.path.includes(elem))
+    }
+    return ret
   }
 
   analyze (value) {

package/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js

@@ -6,7 +6,7 @@ class SqlInjectionAnalyzer extends InjectionAnalyzer {
     super('SQL_INJECTION')
     this.addSub('apm:mysql:query:start', ({ sql }) => this.analyze(sql))
     this.addSub('apm:mysql2:query:start', ({ sql }) => this.analyze(sql))
-    this.addSub('apm:pg:query:start', ({ originalQuery }) => this.analyze(originalQuery))
+    this.addSub('apm:pg:query:start', ({ query }) => this.analyze(query.text))
   }
 }
 

package/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js

@@ -32,12 +32,18 @@ class Analyzer extends Plugin {
     return false
   }
 
+  _isExcluded (location) {
+    return false
+  }
+
   _report (value, context) {
     const evidence = this._getEvidence(value, context)
     const location = this._getLocation()
-    const spanId = context && context.rootSpan && context.rootSpan.context().toSpanId()
-    const vulnerability = createVulnerability(this._type, evidence, spanId, location)
-    addVulnerability(context, vulnerability)
+    if (!this._isExcluded(location)) {
+      const spanId = context && context.rootSpan && context.rootSpan.context().toSpanId()
+      const vulnerability = createVulnerability(this._type, evidence, spanId, location)
+      addVulnerability(context, vulnerability)
+    }
   }
 
   _reportIfVulnerable (value, context) {

package/packages/dd-trace/src/appsec/iast/index.js

@@ -6,6 +6,7 @@ const overheadController = require('./overhead-controller')
 const dc = require('diagnostics_channel')
 const iastContextFunctions = require('./iast-context')
 const { enableTaintTracking, disableTaintTracking, createTransaction, removeTransaction } = require('./taint-tracking')
+
 const telemetryLogs = require('./telemetry/logs')
 const IAST_ENABLED_TAG_KEY = '_dd.iast.enabled'
 
@@ -16,7 +17,7 @@ const requestClose = dc.channel('dd-trace:incomingHttpRequestEnd')
 
 function enable (config, _tracer) {
   enableAllAnalyzers()
-  enableTaintTracking()
+  enableTaintTracking(config.iast)
   requestStart.subscribe(onIncomingHttpRequestStart)
   requestClose.subscribe(onIncomingHttpRequestEnd)
   overheadController.configure(config.iast)

package/packages/dd-trace/src/appsec/iast/taint-tracking/index.js

@@ -1,20 +1,27 @@
 'use strict'
 
 const { enableRewriter, disableRewriter } = require('./rewriter')
-const { createTransaction, removeTransaction, enableTaintOperations, disableTaintOperations } = require('./operations')
+const { createTransaction,
+  removeTransaction,
+  setMaxTransactions,
+  enableTaintOperations,
+  disableTaintOperations } = require('./operations')
+
 const taintTrackingPlugin = require('./plugin')
 
 module.exports = {
-  enableTaintTracking () {
+  enableTaintTracking (config) {
     enableRewriter()
     enableTaintOperations()
     taintTrackingPlugin.enable()
+    setMaxTransactions(config.maxConcurrentRequests)
   },
   disableTaintTracking () {
     disableRewriter()
     disableTaintOperations()
     taintTrackingPlugin.disable()
   },
+  setMaxTransactions: setMaxTransactions,
   createTransaction: createTransaction,
   removeTransaction: removeTransaction
 }

package/packages/dd-trace/src/appsec/iast/taint-tracking/operations.js

@@ -87,6 +87,14 @@ function disableTaintOperations () {
   global._ddiast = TaintTrackingDummy
 }
 
+function setMaxTransactions (transactions) {
+  if (!transactions) {
+    return
+  }
+
+  TaintedUtils.setMaxTransactions(transactions)
+}
+
 module.exports = {
   createTransaction,
   removeTransaction,
@@ -96,5 +104,6 @@ module.exports = {
   getRanges,
   enableTaintOperations,
   disableTaintOperations,
+  setMaxTransactions,
   IAST_TRANSACTION_ID
 }

package/packages/dd-trace/src/appsec/index.js

@@ -1,7 +1,5 @@
 'use strict'
 
-const fs = require('fs')
-const path = require('path')
 const log = require('../log')
 const RuleManager = require('./rule_manager')
 const remoteConfig = require('./remote_config')
@@ -12,7 +10,7 @@ const Reporter = require('./reporter')
 const web = require('../plugins/util/web')
 const { extractIp } = require('../plugins/util/ip_extractor')
 const { HTTP_CLIENT_IP } = require('../../../../ext/tags')
-const { block, loadTemplates, loadTemplatesAsync } = require('./blocking')
+const { block, setTemplates } = require('./blocking')
 
 let isEnabled = false
 let config
@@ -21,21 +19,10 @@ function enable (_config) {
   if (isEnabled) return
 
   try {
-    loadTemplates(_config)
-    const rules = fs.readFileSync(_config.appsec.rules || path.join(__dirname, 'recommended.json'))
-    enableFromRules(_config, JSON.parse(rules))
-  } catch (err) {
-    abortEnable(err)
-  }
-}
+    setTemplates(_config)
 
-async function enableAsync (_config) {
-  if (isEnabled) return
-
-  try {
-    await loadTemplatesAsync(_config)
-    const rules = await fs.promises.readFile(_config.appsec.rules || path.join(__dirname, 'recommended.json'))
-    enableFromRules(_config, JSON.parse(rules))
+    // TODO: inline this function
+    enableFromRules(_config, _config.appsec.rules)
   } catch (err) {
     abortEnable(err)
   }
@@ -169,7 +156,6 @@ function disable () {
 
 module.exports = {
   enable,
-  enableAsync,
   disable,
   incomingHttpStartTranslator,
   incomingHttpEndTranslator

package/packages/dd-trace/src/appsec/recommended.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.5.2"
+    "rules_version": "1.6.0"
   },
   "rules": [
     {
@@ -2907,7 +2907,8 @@
         }
       ],
       "transformers": [
-        "removeNulls"
+        "removeNulls",
+        "urlDecodeUni"
       ]
     },
     {
@@ -2957,7 +2958,8 @@
         }
       ],
       "transformers": [
-        "removeNulls"
+        "removeNulls",
+        "urlDecodeUni"
       ]
     },
     {
@@ -3007,7 +3009,8 @@
         }
       ],
       "transformers": [
-        "removeNulls"
+        "removeNulls",
+        "urlDecodeUni"
       ]
     },
     {
@@ -3054,7 +3057,8 @@
         }
       ],
       "transformers": [
-        "removeNulls"
+        "removeNulls",
+        "urlDecodeUni"
       ]
     },
     {
@@ -3088,8 +3092,7 @@
               ".parentnode",
               ".innerhtml",
               "window.location",
-              "-moz-binding",
-              "<![cdata["
+              "-moz-binding"
             ]
           },
           "operator": "phrase_match"
@@ -3545,7 +3548,7 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "\\b(?i:eval|settimeout|setinterval|new\\s+Function|alert|prompt)\\s*\\([^\\)]",
+            "regex": "\\b(?i:eval|settimeout|setinterval|new\\s+Function|alert|prompt)[\\s+]*\\([^\\)]",
             "options": {
               "case_sensitive": true,
               "min_length": 5
@@ -5347,14 +5350,12 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "(http|https):\\/\\/(?:.*\\.)?(?:burpcollaborator\\.net|localtest\\.me|mail\\.ebc\\.apple\\.com|bugbounty\\.dod\\.network|.*\\.[nx]ip\\.io|oastify\\.com|oast\\.(?:pro|live|site|online|fun|me)|sslip\\.io|requestbin\\.com|requestbin\\.net|hookbin\\.com|webhook\\.site|canarytokens\\.com|interact\\.sh|ngrok\\.io|bugbounty\\.click|prbly\\.win|qualysperiscope\\.com)"
+            "regex": "(http|https):\\/\\/(?:.*\\.)?(?:burpcollaborator\\.net|localtest\\.me|mail\\.ebc\\.apple\\.com|bugbounty\\.dod\\.network|.*\\.[nx]ip\\.io|oastify\\.com|oast\\.(?:pro|live|site|online|fun|me)|sslip\\.io|requestbin\\.com|requestbin\\.net|hookbin\\.com|webhook\\.site|canarytokens\\.com|interact\\.sh|ngrok\\.io|bugbounty\\.click|prbly\\.win|qualysperiscope\\.com|vii.one|act1on3.ru)"
           },
           "operator": "match_regex"
         }
       ],
-      "transformers": [
-        "lowercase"
-      ]
+      "transformers": []
     },
     {
       "id": "sqr-000-015",
@@ -5429,7 +5430,9 @@
           "operator": "match_regex"
         }
       ],
-      "transformers": []
+      "transformers": [
+        "unicode_normalize"
+      ]
     },
     {
       "id": "ua0-600-0xx",
@@ -5905,7 +5908,7 @@
       "tags": {
         "type": "security_scanner",
         "category": "attack_attempt",
-        "confidence": "1"
+        "confidence": "0"
       },
       "conditions": [
         {
@@ -6616,6 +6619,32 @@
         "block"
       ]
     },
+    {
+      "id": "ua0-600-57x",
+      "name": "AlertLogic",
+      "tags": {
+        "type": "security_scanner",
+        "category": "attack_attempt",
+        "confidence": "0"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "user-agent"
+                ]
+              }
+            ],
+            "regex": "\\bAlertLogic-MDR-"
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": []
+    },
     {
       "id": "ua0-600-5xx",
       "name": "Blind SQL Injection Brute Forcer",

package/packages/dd-trace/src/appsec/remote_config/index.js

@@ -23,7 +23,7 @@ function enable (config) {
         }
 
         if (shouldEnable) {
-          require('..').enableAsync(config).catch(() => {})
+          require('..').enable(config)
         } else {
           require('..').disable()
         }

package/packages/dd-trace/src/appsec/sdk/index.js

@@ -2,14 +2,14 @@
 
 const { trackUserLoginSuccessEvent, trackUserLoginFailureEvent, trackCustomEvent } = require('./track_event')
 const { checkUserAndSetUser, blockRequest } = require('./user_blocking')
-const { loadTemplates } = require('../blocking')
+const { setTemplates } = require('../blocking')
 const { setUser } = require('./set_user')
 
 class AppsecSdk {
   constructor (tracer, config) {
     this._tracer = tracer
     if (config) {
-      loadTemplates(config)
+      setTemplates(config)
     }
   }
 

package/packages/dd-trace/src/config.js

@@ -9,7 +9,6 @@ const coalesce = require('koalas')
 const tagger = require('./tagger')
 const { isTrue, isFalse } = require('./util')
 const uuid = require('crypto-randomuuid')
-const path = require('path')
 
 const fromEntries = Object.fromEntries || (entries =>
   entries.reduce((obj, [k, v]) => Object.assign(obj, { [k]: v }), {}))
@@ -22,16 +21,7 @@ function maybeFile (filepath) {
   try {
     return fs.readFileSync(filepath, 'utf8')
   } catch (e) {
-    return undefined
-  }
-}
-
-function maybePath (filepath) {
-  if (!filepath) return
-  try {
-    fs.openSync(filepath, 'r')
-    return filepath
-  } catch (e) {
+    log.error(e)
     return undefined
   }
 }
@@ -282,6 +272,18 @@ class Config {
       false
     )
 
+    const DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED = coalesce(
+      options.traceId128BitGenerationEnabled,
+      process.env.DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED,
+      false
+    )
+
+    const DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED = coalesce(
+      options.traceId128BitLoggingEnabled,
+      process.env.DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED,
+      false
+    )
+
     let appsec = options.appsec != null ? options.appsec : options.experimental && options.experimental.appsec
 
     if (typeof appsec === 'boolean') {
@@ -326,14 +328,12 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
 |[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}`
     )
     const DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML = coalesce(
-      maybePath(appsec.blockedTemplateHtml),
-      maybePath(process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML),
-      path.join(__dirname, 'appsec', 'templates', 'blocked.html')
+      maybeFile(appsec.blockedTemplateHtml),
+      maybeFile(process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML)
     )
     const DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON = coalesce(
-      maybePath(appsec.blockedTemplateJson),
-      maybePath(process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON),
-      path.join(__dirname, 'appsec', 'templates', 'blocked.json')
+      maybeFile(appsec.blockedTemplateJson),
+      maybeFile(process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON)
     )
 
     const inAWSLambda = process.env.AWS_LAMBDA_FUNCTION_NAME !== undefined
@@ -479,7 +479,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     this.tagsHeaderMaxLength = parseInt(DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH)
     this.appsec = {
       enabled: DD_APPSEC_ENABLED,
-      rules: DD_APPSEC_RULES,
+      rules: DD_APPSEC_RULES ? safeJsonParse(maybeFile(DD_APPSEC_RULES)) : require('./appsec/recommended.json'),
       rateLimit: DD_APPSEC_TRACE_RATE_LIMIT,
       wafTimeout: DD_APPSEC_WAF_TIMEOUT,
       obfuscatorKeyRegex: DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP,
@@ -509,6 +509,9 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       enabled: isTrue(DD_TRACE_STATS_COMPUTATION_ENABLED)
     }
 
+    this.traceId128BitGenerationEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED)
+    this.traceId128BitLoggingEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED)
+
     tagger.add(this.tags, {
       service: this.service,
       env: this.env,

package/packages/dd-trace/src/datastreams/encoding.js

@@ -16,13 +16,13 @@ function encodeVarint (v) {
 // decodes positive and negative numbers, using zig zag encoding to reduce the size of the variable length encoding.
 // uses high and low part to ensure those parts are under the limit for byte operations in javascript (32 bits)
 function decodeVarint (b) {
-  const [low, high] = decodeUvarint64(b)
+  const [low, high, bytes] = decodeUvarint64(b)
   if (low === undefined || high === undefined) {
-    return undefined
+    return [undefined, bytes]
   }
   const positive = (low & 1) === 0
   const abs = (low >>> 1) + high * 0x80000000
-  return positive ? abs : -abs
+  return [positive ? abs : -abs, bytes]
 }
 
 const maxVarLen64 = 9
@@ -50,7 +50,7 @@ function decodeUvarint64 (
   let s = 0
   for (let i = 0; ; i++) {
     if (bytes.length <= i) {
-      return [undefined, undefined]
+      return [undefined, undefined, bytes.slice(bytes.length)]
     }
     const n = bytes[i]
     if (n < 0x80 || i === maxVarLen64 - 1) {
@@ -61,7 +61,7 @@ function decodeUvarint64 (
       if (s > 0) {
         high |= s - 32 > 0 ? n << (s - 32) : n >> (32 - s)
       }
-      return [low, high]
+      return [low, high, bytes]
     }
     if (s < 32) {
       low |= (n & 0x7f) << s

package/packages/dd-trace/src/dcitm.js

@@ -0,0 +1,51 @@
+'use strict'
+
+const dc = require('diagnostics_channel')
+
+const CHANNEL_PREFIX = 'dd-trace:bundledModuleLoadStart'
+
+if (!dc.subscribe) {
+  dc.subscribe = (channel, cb) => {
+    dc.channel(channel).subscribe(cb)
+  }
+}
+if (!dc.unsubscribe) {
+  dc.unsubscribe = (channel, cb) => {
+    if (dc.channel(channel).hasSubscribers) {
+      dc.channel(channel).unsubscribe(cb)
+    }
+  }
+}
+
+module.exports = DcitmHook
+
+/**
+ * This allows for listening to diagnostic channel events when a module is loaded.
+ * Currently it's intended use is for situations like when code runs through a bundler.
+ *
+ * Unlike RITM and IITM, which have files available on a filesystem at runtime, DCITM
+ * requires access to a package's version ahead of time as the package.json file likely
+ * won't be available.
+ *
+ * This function runs many times at startup, once for every module that dd-trace may trace.
+ * As it runs on a per-module basis we're creating per-module channels.
+ */
+function DcitmHook (moduleNames, options, onrequire) {
+  if (!(this instanceof DcitmHook)) return new DcitmHook(moduleNames, options, onrequire)
+
+  function onModuleLoad (payload) {
+    payload.module = onrequire(payload.module, payload.path, undefined, payload.version)
+  }
+
+  for (const moduleName of moduleNames) {
+    // dc.channel(`${CHANNEL_PREFIX}:${moduleName}`).subscribe(onModuleLoad)
+    dc.subscribe(`${CHANNEL_PREFIX}:${moduleName}`, onModuleLoad)
+  }
+
+  this.unhook = function dcitmUnload () {
+    for (const moduleName of moduleNames) {
+      // dc.channel(`${CHANNEL_PREFIX}:${moduleName}`).unsubscribe(onModuleLoad)
+      dc.unsubscribe(`${CHANNEL_PREFIX}:${moduleName}`, onModuleLoad)
+    }
+  }
+}

package/packages/dd-trace/src/lambda/handler.js

@@ -50,7 +50,7 @@ function crashFlush () {
       [ERROR_TYPE]: error.name
     })
   } else {
-    log.warn('An impending timeout was reached, but no root span was found. No error will be tagged.')
+    log.debug('An impending timeout was reached, but no root span was found. No error will be tagged.')
   }
 
   tracer._processor.killAll()
@@ -59,6 +59,23 @@ function crashFlush () {
   }
 }
 
+/**
+ * Extracts the context from the given Lambda handler arguments.
+ *
+ * @param {*} args any amount of arguments
+ * @returns the context, if extraction was succesful.
+ */
+function extractContext (args) {
+  let context = args.length > 1 ? args[1] : undefined
+  if (context === undefined || context.getRemainingTimeInMillis === undefined) {
+    context = args.length > 2 ? args[2] : undefined
+    if (context === undefined || context.getRemainingTimeInMillis === undefined) {
+      throw Error('Could not extract context')
+    }
+  }
+  return context
+}
+
 /**
  * Patches your AWS Lambda handler function to add some tracing support.
  *
@@ -66,14 +83,21 @@ function crashFlush () {
  */
 exports.datadog = function datadog (lambdaHandler) {
   return (...args) => {
-    const context = args[1]
     const patched = lambdaHandler.apply(this, args)
-    checkTimeout(context)
 
-    if (patched) {
-      // clear the timeout as soon as a result is returned
-      patched.then(_ => clearTimeout(__lambdaTimeout))
+    try {
+      const context = extractContext(args)
+
+      checkTimeout(context)
+
+      if (patched) {
+        // clear the timeout as soon as a result is returned
+        patched.then(_ => clearTimeout(__lambdaTimeout))
+      }
+    } catch (e) {
+      log.debug('Error patching AWS Lambda handler. Timeout spans will not be generated.')
     }
+
     return patched
   }
 }

package/packages/dd-trace/src/opentracing/propagation/log.js

@@ -14,7 +14,12 @@ class LogPropagator {
     carrier.dd = {}
 
     if (spanContext) {
-      carrier.dd.trace_id = spanContext.toTraceId()
+      if (this._config.traceId128BitLoggingEnabled && spanContext._trace.tags['_dd.p.tid']) {
+        carrier.dd.trace_id = spanContext._trace.tags['_dd.p.tid'] + spanContext._traceId.toString(16)
+      } else {
+        carrier.dd.trace_id = spanContext.toTraceId()
+      }
+
       carrier.dd.span_id = spanContext.toSpanId()
     }
 
@@ -28,12 +33,23 @@ class LogPropagator {
       return null
     }
 
-    const spanContext = new DatadogSpanContext({
-      traceId: id(carrier.dd.trace_id, 10),
-      spanId: id(carrier.dd.span_id, 10)
-    })
-
-    return spanContext
+    if (carrier.dd.trace_id.length === 32) {
+      const hi = carrier.dd.trace_id.substring(0, 16)
+      const lo = carrier.dd.trace_id.substring(16, 32)
+      const spanContext = new DatadogSpanContext({
+        traceId: id(lo, 16),
+        spanId: id(carrier.dd.span_id, 10)
+      })
+
+      spanContext._trace.tags['_dd.p.tid'] = hi
+
+      return spanContext
+    } else {
+      return new DatadogSpanContext({
+        traceId: id(carrier.dd.trace_id, 10),
+        spanId: id(carrier.dd.span_id, 10)
+      })
+    }
   }
 }
 

package/packages/dd-trace/src/opentracing/propagation/text_map.js

@@ -132,7 +132,7 @@ class TextMapPropagator {
     const hasB3multi = this._hasPropagationStyle('inject', 'b3multi')
     if (!(hasB3 || hasB3multi)) return
 
-    carrier[b3TraceKey] = spanContext._traceId.toString(16)
+    carrier[b3TraceKey] = this._getB3TraceId(spanContext)
     carrier[b3SpanKey] = spanContext._spanId.toString(16)
     carrier[b3SampledKey] = spanContext._sampling.priority >= AUTO_KEEP ? '1' : '0'
 
@@ -149,7 +149,7 @@ class TextMapPropagator {
     const hasB3SingleHeader = this._hasPropagationStyle('inject', 'b3 single header')
     if (!hasB3SingleHeader) return null
 
-    const traceId = spanContext._traceId.toString(16)
+    const traceId = this._getB3TraceId(spanContext)
     const spanId = spanContext._spanId.toString(16)
     const sampled = spanContext._sampling.priority >= AUTO_KEEP ? '1' : '0'
 
@@ -277,6 +277,8 @@ class TextMapPropagator {
       spanContext._sampling.priority = priority
     }
 
+    this._extract128BitTraceId(b3[b3TraceKey], spanContext)
+
     return spanContext
   }
 
@@ -321,6 +323,8 @@ class TextMapPropagator {
         tracestate
       })
 
+      this._extract128BitTraceId(traceId, spanContext)
+
       tracestate.forVendor('dd', state => {
         for (const [key, value] of state.entries()) {
           switch (key) {
@@ -484,6 +488,20 @@ class TextMapPropagator {
     }
   }
 
+  _extract128BitTraceId (traceId, spanContext) {
+    if (!spanContext) return
+
+    const buffer = spanContext._traceId.toBuffer()
+
+    if (buffer.length !== 16) return
+
+    const tid = traceId.substring(0, 16)
+
+    if (tid === '0000000000000000') return
+
+    spanContext._trace.tags['_dd.p.tid'] = tid
+  }
+
   _validateTagKey (key) {
     return tagKeyExpr.test(key)
   }
@@ -501,6 +519,14 @@ class TextMapPropagator {
       return AUTO_REJECT
     }
   }
+
+  _getB3TraceId (spanContext) {
+    if (spanContext._traceId.toBuffer().length <= 8 && spanContext._trace.tags['_dd.p.tid']) {
+      return spanContext._trace.tags['_dd.p.tid'] + spanContext._traceId.toString(16)
+    }
+
+    return spanContext._traceId.toString(16)
+  }
 }
 
 module.exports = TextMapPropagator

package/packages/dd-trace/src/opentracing/span.js

@@ -39,7 +39,7 @@ class DatadogSpan {
     // This is necessary for span count metrics.
     this._name = operationName
 
-    this._spanContext = this._createContext(parent)
+    this._spanContext = this._createContext(parent, fields)
     this._spanContext._name = operationName
     this._spanContext._tags = tags
     this._spanContext._hostname = hostname
@@ -145,7 +145,7 @@ class DatadogSpan {
     this._processor.process(this)
   }
 
-  _createContext (parent) {
+  _createContext (parent, fields) {
     let spanContext
 
     if (parent) {
@@ -158,16 +158,27 @@ class DatadogSpan {
         trace: parent._trace,
         tracestate: parent._tracestate
       })
+
+      if (!spanContext._trace.startTime) {
+        spanContext._trace.startTime = dateNow()
+      }
     } else {
       const spanId = id()
+      const startTime = dateNow()
       spanContext = new SpanContext({
         traceId: spanId,
         spanId
       })
+      spanContext._trace.startTime = startTime
+
+      if (fields.traceId128BitGenerationEnabled) {
+        spanContext._trace.tags['_dd.p.tid'] = Math.floor(startTime / 1000).toString(16)
+          .padStart(8, '0')
+          .padEnd(16, '0')
+      }
     }
 
     spanContext._trace.started.push(this)
-    spanContext._trace.startTime = spanContext._trace.startTime || dateNow()
     spanContext._trace.ticks = spanContext._trace.ticks || now()
 
     return spanContext

package/packages/dd-trace/src/opentracing/span_context.js

@@ -34,7 +34,9 @@ class DatadogSpanContext {
 
   toTraceparent () {
     const flags = this._sampling.priority >= AUTO_KEEP ? '01' : '00'
-    const traceId = this._traceId.toString(16).padStart(32, '0')
+    const traceId = this._traceId.toBuffer().length <= 8 && this._trace.tags['_dd.p.tid']
+      ? this._trace.tags['_dd.p.tid'] + this._traceId.toString(16).padStart(16, '0')
+      : this._traceId.toString(16).padStart(32, '0')
     const spanId = this._spanId.toString(16).padStart(16, '0')
     const version = (this._traceparent && this._traceparent.version) || '00'
     return `${version}-${traceId}-${spanId}-${flags}`

package/packages/dd-trace/src/opentracing/tracer.js

@@ -33,6 +33,7 @@ class DatadogTracer {
     this._processor = new SpanProcessor(this._exporter, this._prioritySampler, config)
     this._url = this._exporter._url
     this._enableGetRumData = config.experimental.enableGetRumData
+    this._traceId128BitGenerationEnabled = config.traceId128BitGenerationEnabled
     this._propagators = {
       [formats.TEXT_MAP]: new TextMapPropagator(config),
       [formats.HTTP_HEADERS]: new HttpPropagator(config),
@@ -58,7 +59,8 @@ class DatadogTracer {
       parent,
       tags,
       startTime: options.startTime,
-      hostname: this._hostname
+      hostname: this._hostname,
+      traceId128BitGenerationEnabled: this._traceId128BitGenerationEnabled
     }, this._debug)
 
     span.addTags(this._tags)

package/scripts/install_plugin_modules.js

@@ -89,6 +89,8 @@ async function assertInstrumentation (instrumentation, external) {
 }
 
 async function assertModules (name, version, external) {
+  const range = process.env.RANGE
+  if (range && !semver.subset(version, range)) return
   addFolder(name)
   addFolder(name, version)
   assertFolder(name)

package/packages/dd-trace/src/appsec/templates/blocked.json

@@ -1,8 +0,0 @@
-{
-  "errors": [
-    {
-      "title": "You've been blocked",
-      "detail": "Sorry, you cannot access this page. Please contact the customer service team. Security provided by Datadog."
-    }
-  ]
-}