dd-trace 3.1.0 → 3.2.0

package/LICENSE-3rdparty.csv

@@ -3,6 +3,7 @@ require,@datadog/native-appsec,Apache license 2.0,Copyright 2018 Datadog Inc.
 require,@datadog/native-metrics,Apache license 2.0,Copyright 2018 Datadog Inc.
 require,@datadog/pprof,Apache license 2.0,Copyright 2019 Google Inc.
 require,@datadog/sketches-js,Apache license 2.0,Copyright 2020 Datadog Inc.
+require,cidr-matcher,MIT,Copyright 2015 Marco Pracucci
 require,crypto-randomuuid,MIT,Copyright 2021 Node.js Foundation and contributors
 require,diagnostics_channel,MIT,Copyright 2021 Simon D.
 require,ignore,MIT,Copyright 2013 Kael Zhang and contributors

package/ext/tags.d.ts

@@ -15,7 +15,8 @@ declare const tags: {
   HTTP_ROUTE: 'http.route'
   HTTP_REQUEST_HEADERS: 'http.request.headers'
   HTTP_RESPONSE_HEADERS: 'http.response.headers'
-  HTTP_USERAGENT: 'http.useragent'
+  HTTP_USERAGENT: 'http.useragent',
+  HTTP_CLIENT_IP: 'http.client_ip'
 }
 
 export = tags

package/ext/tags.js

@@ -20,7 +20,8 @@ const tags = {
   HTTP_ROUTE: 'http.route',
   HTTP_REQUEST_HEADERS: 'http.request.headers',
   HTTP_RESPONSE_HEADERS: 'http.response.headers',
-  HTTP_USERAGENT: 'http.useragent'
+  HTTP_USERAGENT: 'http.useragent',
+  HTTP_CLIENT_IP: 'http.client_ip'
 }
 
 // Deprecated

package/index.d.ts

@@ -267,6 +267,22 @@ export declare interface TracerOptions {
    */
   sampleRate?: number;
 
+  /**
+   * Global rate limit that is applied on the global sample rate and all rules,
+   * and controls the ingestion rate limit between the agent and the backend.
+   * Defaults to deferring the decision to the agent.
+   */
+  rateLimit?: Number,
+
+  /**
+   * Sampling rules to apply to priority samplin. Each rule is a JSON,
+   * consisting of `service` and `name`, which are regexes to match against
+   * a trace's `service` and `name`, and a corresponding `sampleRate`. If not
+   * specified, will defer to global sampling rate for all spans.
+   * @default []
+   */
+  samplingRules?: SamplingRule[]
+
   /**
    * Interval in milliseconds at which the tracer will submit traces to the agent.
    * @default 2000
@@ -298,7 +314,10 @@ export declare interface TracerOptions {
   protocolVersion?: string
 
   /**
-   * Configuration of the ingestion between the agent and the backend.
+   * Deprecated in favor of the global versions of the variables provided under this option
+   *
+   * @deprecated
+   * @hidden
    */
   ingestion?: {
     /**
@@ -307,7 +326,7 @@ export declare interface TracerOptions {
     sampleRate?: number
 
     /**
-     * Controls the ingestion rate limit between the agent and the backend.
+     * Controls the ingestion rate limit between the agent and the backend. Defaults to deferring the decision to the agent.
      */
     rateLimit?: number
   };
@@ -333,32 +352,36 @@ export declare interface TracerOptions {
     exporter?: 'log' | 'agent'
 
     /**
-     * Configuration of the priority sampler. Supports a global config and rules by span name or service name. The first matching rule is applied, and if no rule matches it falls back to the global config or on the rates provided by the agent if there is no global config.
+     * Whether to enable the experimental `getRumData` method.
+     * @default false
      */
-    sampler?: {
+    enableGetRumData?: boolean
+
+    /**
+     * Configuration of the IAST. Can be a boolean as an alias to `iast.enabled`.
+     */
+    iast?: boolean  | {
       /**
-       * Sample rate to apply globally when no other rule is matched. Omit to fallback on the dynamic rates returned by the agent instead.
+       * Whether to enable IAST.
+       * @default false
        */
-      sampleRate?: Number,
-
+      enabled?: boolean,
       /**
-       * Global rate limit that is applied on the global sample rate and all rules.
-       * @default 100
+       * Controls the percentage of requests that iast will analyze
+       * @default 30
        */
-      rateLimit?: Number,
-
+      requestSampling?: number,
       /**
-       * Sampling rules to apply to priority sampling.
-       * @default []
+       * Controls how many request can be analyzing code vulnerabilities at the same time
+       * @default 2
        */
-      rules?: SamplingRule[]
+      maxConcurrentRequests?: number,
+      /**
+       * Controls how many code vulnerabilities can be detected in the same request
+       * @default 2
+       */
+      maxContextOperations?: number
     }
-
-    /**
-     * Whether to enable the experimental `getRumData` method.
-     * @default false
-     */
-    enableGetRumData?: boolean
   };
 
   /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -61,11 +61,12 @@
     "@datadog/native-appsec": "^1.2.1",
     "@datadog/native-metrics": "^1.4.2",
     "@datadog/pprof": "^1.0.2",
-    "@datadog/sketches-js": "^2.0.0",
+    "@datadog/sketches-js": "^2.1.0",
+    "cidr-matcher": "^2.1.1",
     "crypto-randomuuid": "^1.0.0",
     "diagnostics_channel": "^1.1.0",
     "ignore": "^5.2.0",
-    "import-in-the-middle": "^1.3.0",
+    "import-in-the-middle": "^1.3.1",
     "istanbul-lib-coverage": "3.2.0",
     "koalas": "^1.0.2",
     "limiter": "^1.1.4",

package/packages/datadog-instrumentations/src/crypto.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const {
+  channel,
+  addHook
+} = require('./helpers/instrument')
+const shimmer = require('../../datadog-shimmer')
+
+const cryptoCh = channel('datadog:crypto:hashing:start')
+
+addHook({ name: 'crypto' }, crypto => {
+  shimmer.massWrap(
+    crypto,
+    ['createHash', 'createHmac', 'createSign', 'createVerify', 'sign', 'verify'],
+    wrapMethod
+  )
+  return crypto
+})
+
+function wrapMethod (cryptoMethod) {
+  return function () {
+    if (cryptoCh.hasSubscribers) {
+      if (arguments.length > 0) {
+        const algorithm = arguments[0]
+        cryptoCh.publish({ algorithm })
+      }
+    }
+    return cryptoMethod.apply(this, arguments)
+  }
+}

package/packages/datadog-instrumentations/src/helpers/hooks.js

@@ -17,6 +17,7 @@ module.exports = {
   'cassandra-driver': () => require('../cassandra-driver'),
   'connect': () => require('../connect'),
   'couchbase': () => require('../couchbase'),
+  'crypto': () => require('../crypto'),
   'cypress': () => require('../cypress'),
   'dns': () => require('../dns'),
   'elasticsearch': () => require('../elasticsearch'),

package/packages/datadog-instrumentations/src/http/server.js

@@ -29,7 +29,7 @@ function wrapResponseEmit (emit) {
       return emit.apply(this, arguments)
     }
 
-    if (eventName === 'finish') {
+    if (eventName === 'close') {
       finishServerCh.publish({ req: this.req })
     }
 

package/packages/datadog-instrumentations/src/net.js

@@ -49,6 +49,19 @@ addHook({ name: 'net' }, net => {
         setupListeners(this, 'tcp', asyncResource)
       }
 
+      const emit = this.emit
+      this.emit = function (eventName) {
+        switch (eventName) {
+          case 'ready':
+          case 'connect':
+            return callbackResource.runInAsyncScope(() => {
+              return emit.apply(this, arguments)
+            })
+          default:
+            return emit.apply(this, arguments)
+        }
+      }
+
       try {
         return connect.apply(this, arguments)
       } catch (err) {

package/packages/datadog-plugin-mongodb-core/src/index.js

@@ -73,21 +73,21 @@ function truncate (input) {
   return input.slice(0, Math.min(input.length, 10000))
 }
 
-function simplify (input) {
-  return isBSON(input) ? input.toHexString() : input
-}
-
 function shouldSimplify (input) {
-  return !isObject(input) || isBSON(input)
+  return !isObject(input)
 }
 
 function shouldHide (input) {
-  return Buffer.isBuffer(input) || typeof input === 'function'
+  return Buffer.isBuffer(input) || typeof input === 'function' || isBinary(input)
 }
 
 function limitDepth (input) {
+  if (isBSON(input)) {
+    input = input.toJSON()
+  }
+
   if (shouldHide(input)) return '?'
-  if (shouldSimplify(input)) return simplify(input)
+  if (shouldSimplify(input)) return input
 
   const output = {}
   const queue = [{
@@ -104,11 +104,16 @@ function limitDepth (input) {
     for (const key in input) {
       if (typeof input[key] === 'function') continue
 
-      const child = input[key]
+      let child = input[key]
+
+      if (isBSON(child)) {
+        child = child.toJSON()
+      }
+
       if (depth >= 10 || shouldHide(child)) {
         output[key] = '?'
       } else if (shouldSimplify(child)) {
-        output[key] = simplify(child)
+        output[key] = child
       } else {
         queue.push({
           input: child,
@@ -127,7 +132,11 @@ function isObject (val) {
 }
 
 function isBSON (val) {
-  return val && val._bsontype
+  return val && val._bsontype && !isBinary(val)
+}
+
+function isBinary (val) {
+  return val && val._bsontype === 'Binary'
 }
 
 module.exports = MongodbCorePlugin

package/packages/dd-trace/src/appsec/iast/analyzers/analyzers.js

@@ -0,0 +1,3 @@
+module.exports = {
+  'WEAK_HASH_ANALYZER': require('./weak-hash-analyzer')
+}

package/packages/dd-trace/src/appsec/iast/analyzers/index.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const analyzers = require('./analyzers')
+
+function enableAllAnalyzers () {
+  for (const analyzer in analyzers) {
+    analyzers[analyzer].configure(true)
+  }
+}
+
+function disableAllAnalyzers () {
+  for (const analyzer in analyzers) {
+    analyzers[analyzer].configure(false)
+  }
+}
+
+module.exports = {
+  enableAllAnalyzers,
+  disableAllAnalyzers
+}

package/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js

@@ -0,0 +1,48 @@
+'use strict'
+
+const Plugin = require('../../../../src/plugins/plugin')
+const { storage } = require('../../../../../datadog-core')
+const { getFirstNonDDPathAndLine } = require('./../path-line')
+const { createVulnerability, addVulnerability } = require('../vulnerability-reporter')
+const { getIastContext } = require('../iast-context')
+const overheadController = require('../overhead-controller')
+
+class Analyzer extends Plugin {
+  constructor (type) {
+    super()
+    this._type = type
+  }
+
+  _isVulnerable (value, context) {
+    return false
+  }
+
+  _report (value, context) {
+    const evidence = this._getEvidence(value)
+    const location = this._getLocation()
+    const spanId = context && context.rootSpan && context.rootSpan.context().toSpanId()
+    const vulnerability = createVulnerability(this._type, evidence, spanId, location)
+    addVulnerability(context, vulnerability)
+  }
+
+  _getEvidence (value) {
+    return { value }
+  }
+
+  _getLocation () {
+    return getFirstNonDDPathAndLine()
+  }
+
+  analyze (value) {
+    const iastContext = getIastContext(storage.getStore())
+    if (iastContext && this._isVulnerable(value, iastContext) && this._checkOCE(iastContext)) {
+      this._report(value, iastContext)
+    }
+  }
+
+  _checkOCE (context) {
+    return overheadController.hasQuota(overheadController.OPERATIONS.REPORT_VULNERABILITY, context)
+  }
+}
+
+module.exports = Analyzer

package/packages/dd-trace/src/appsec/iast/analyzers/weak-hash-analyzer.js

@@ -0,0 +1,24 @@
+'use strict'
+const Analyzer = require('./vulnerability-analyzer')
+
+const INSECURE_HASH_ALGORITHMS = new Set([
+  'md4', 'md4WithRSAEncryption', 'RSA-MD4',
+  'RSA-MD5', 'md5', 'md5-sha1', 'ssl3-md5', 'md5WithRSAEncryption',
+  'RSA-SHA1', 'RSA-SHA1-2', 'sha1', 'md5-sha1', 'sha1WithRSAEncryption', 'ssl3-sha1'
+].map(algorithm => algorithm.toLowerCase()))
+
+class WeakHashAnalyzer extends Analyzer {
+  constructor () {
+    super('WEAK_HASH')
+    this.addSub('datadog:crypto:hashing:start', ({ algorithm }) => this.analyze(algorithm))
+  }
+
+  _isVulnerable (algorithm) {
+    if (typeof algorithm === 'string') {
+      return INSECURE_HASH_ALGORITHMS.has(algorithm.toLowerCase())
+    }
+    return false
+  }
+}
+
+module.exports = new WeakHashAnalyzer()

package/packages/dd-trace/src/appsec/iast/iast-context.js

@@ -0,0 +1,50 @@
+const IAST_CONTEXT_KEY = Symbol('_dd.iast.context')
+
+function getIastContext (store) {
+  return store && store[IAST_CONTEXT_KEY]
+}
+
+/* TODO Fix storage problem when the close event is called without
+        finish event to remove `topContext` references
+  We have to save the context in two places, because
+  clean can be called when the storage store is not available
+ */
+function saveIastContext (store, topContext, context) {
+  if (store && topContext) {
+    store[IAST_CONTEXT_KEY] = context
+    topContext[IAST_CONTEXT_KEY] = context
+    return store[IAST_CONTEXT_KEY]
+  }
+}
+
+/* TODO Fix storage problem when the close event is called without
+        finish event to remove `topContext` references
+  iastContext is currently saved in store and request rootContext
+  to fix problems with `close` without `finish` events
+*/
+function cleanIastContext (store, context, iastContext) {
+  if (store) {
+    if (!iastContext) {
+      iastContext = store[IAST_CONTEXT_KEY]
+    }
+    store[IAST_CONTEXT_KEY] = null
+  }
+  if (context) {
+    if (!iastContext) {
+      iastContext = context[IAST_CONTEXT_KEY]
+    }
+    context[IAST_CONTEXT_KEY] = null
+  }
+  if (iastContext) {
+    Object.keys(iastContext).forEach(key => delete iastContext[key])
+    return true
+  }
+  return false
+}
+
+module.exports = {
+  getIastContext,
+  saveIastContext,
+  cleanIastContext,
+  IAST_CONTEXT_KEY
+}

package/packages/dd-trace/src/appsec/iast/index.js

@@ -0,0 +1,59 @@
+const { sendVulnerabilities } = require('./vulnerability-reporter')
+const { enableAllAnalyzers, disableAllAnalyzers } = require('./analyzers')
+const web = require('../../plugins/util/web')
+const { storage } = require('../../../../datadog-core')
+const overheadController = require('./overhead-controller')
+const dc = require('diagnostics_channel')
+const { saveIastContext, getIastContext, cleanIastContext } = require('./iast-context')
+
+// TODO Change to `apm:http:server:request:[start|close]` when the subscription
+//  order of the callbacks can be enforce
+const requestStart = dc.channel('dd-trace:incomingHttpRequestStart')
+const requestClose = dc.channel('dd-trace:incomingHttpRequestEnd')
+
+function enable (config) {
+  enableAllAnalyzers()
+  requestStart.subscribe(onIncomingHttpRequestStart)
+  requestClose.subscribe(onIncomingHttpRequestEnd)
+  overheadController.configure(config.iast)
+}
+
+function disable () {
+  disableAllAnalyzers()
+  if (requestStart.hasSubscribers) requestStart.unsubscribe(onIncomingHttpRequestStart)
+  if (requestClose.hasSubscribers) requestClose.unsubscribe(onIncomingHttpRequestEnd)
+}
+
+function onIncomingHttpRequestStart (data) {
+  if (data && data.req) {
+    const store = storage.getStore()
+    if (store) {
+      const topContext = web.getContext(data.req)
+      if (topContext) {
+        const rootSpan = topContext.span
+        const isRequestAcquired = overheadController.acquireRequest(rootSpan)
+        if (isRequestAcquired) {
+          const iastContext = saveIastContext(store, topContext, { rootSpan, req: data.req })
+          overheadController.initializeRequestContext(iastContext)
+        }
+      }
+    }
+  }
+}
+
+function onIncomingHttpRequestEnd (data) {
+  if (data && data.req) {
+    const store = storage.getStore()
+    const iastContext = getIastContext(storage.getStore())
+    if (iastContext && iastContext.rootSpan) {
+      overheadController.releaseRequest()
+      sendVulnerabilities(iastContext, iastContext.rootSpan)
+    }
+    // TODO web.getContext(data.req) is required when the request is aborted
+    if (cleanIastContext(store, web.getContext(data.req), iastContext)) {
+      overheadController.releaseRequest()
+    }
+  }
+}
+
+module.exports = { enable, disable, onIncomingHttpRequestEnd, onIncomingHttpRequestStart }

package/packages/dd-trace/src/appsec/iast/overhead-controller.js

@@ -0,0 +1,94 @@
+'use strict'
+
+const OVERHEAD_CONTROLLER_CONTEXT_KEY = 'oce'
+const REPORT_VULNERABILITY = 'REPORT_VULNERABILITY'
+
+const GLOBAL_OCE_CONTEXT = {}
+let config = {}
+let availableRequest = 0
+const OPERATIONS = {
+  REPORT_VULNERABILITY: {
+    hasQuota: (context) => {
+      const reserved = context && context.tokens && context.tokens[REPORT_VULNERABILITY] > 0
+      if (reserved) {
+        context.tokens[REPORT_VULNERABILITY]--
+      }
+      return reserved
+    },
+    name: REPORT_VULNERABILITY,
+    initialTokenBucketSize () {
+      return typeof config.maxContextOperations === 'number' ? config.maxContextOperations : 2
+    },
+    initContext: function (context) {
+      context.tokens[REPORT_VULNERABILITY] = this.initialTokenBucketSize()
+    }
+  }
+}
+
+function _getNewContext () {
+  const oceContext = {
+    tokens: {}
+  }
+
+  for (const operation in OPERATIONS) {
+    OPERATIONS[operation].initContext(oceContext)
+  }
+
+  return oceContext
+}
+
+function _getContext (iastContext) {
+  if (iastContext && iastContext[OVERHEAD_CONTROLLER_CONTEXT_KEY]) {
+    return iastContext[OVERHEAD_CONTROLLER_CONTEXT_KEY]
+  }
+  return GLOBAL_OCE_CONTEXT
+}
+
+function _resetGlobalContext () {
+  Object.assign(GLOBAL_OCE_CONTEXT, _getNewContext())
+}
+
+function acquireRequest (rootSpan) {
+  if (availableRequest > 0) {
+    const sampling = config && typeof config.requestSampling === 'number'
+      ? config.requestSampling : 30
+    if (rootSpan.context().toSpanId().slice(-2) <= sampling) {
+      availableRequest--
+      return true
+    }
+  }
+  return false
+}
+
+function releaseRequest () {
+  if (availableRequest < config.maxConcurrentRequests) {
+    availableRequest++
+  }
+}
+
+function hasQuota (operation, iastContext) {
+  const oceContext = _getContext(iastContext)
+  return operation.hasQuota(oceContext)
+}
+
+function initializeRequestContext (iastContext) {
+  if (iastContext) iastContext[OVERHEAD_CONTROLLER_CONTEXT_KEY] = _getNewContext()
+}
+
+function configure (cfg) {
+  config = cfg
+  availableRequest = config.maxConcurrentRequests
+}
+
+_resetGlobalContext()
+
+module.exports = {
+  OVERHEAD_CONTROLLER_CONTEXT_KEY,
+  OPERATIONS,
+  _resetGlobalContext,
+  initializeRequestContext,
+  hasQuota,
+  acquireRequest,
+  releaseRequest,
+  configure
+}

package/packages/dd-trace/src/appsec/iast/path-line.js

@@ -0,0 +1,70 @@
+const path = require('path')
+const pathLine = {
+  getFirstNonDDPathAndLine,
+  getFirstNonDDPathAndLineFromCallsites, // Exported only for test purposes
+  calculateDDBasePath, // Exported only for test purposes
+  ddBasePath: calculateDDBasePath(__dirname) // Only for test purposes
+}
+
+const EXCLUDED_PATHS = [
+  '/node_modules/diagnostics_channel'
+]
+const EXCLUDED_PATH_PREFIXES = [
+  'node:diagnostics_channel',
+  'diagnostics_channel'
+]
+
+function calculateDDBasePath (dirname) {
+  const dirSteps = dirname.split(path.sep)
+  const packagesIndex = dirSteps.indexOf('packages')
+  return dirSteps.slice(0, packagesIndex).join(path.sep) + path.sep
+}
+
+function getCallSiteInfo () {
+  const previousPrepareStackTrace = Error.prepareStackTrace
+  let callsiteList
+  Error.prepareStackTrace = function (_, callsites) {
+    callsiteList = callsites
+  }
+  const e = new Error()
+  e.stack
+  Error.prepareStackTrace = previousPrepareStackTrace
+  return callsiteList
+}
+
+function getFirstNonDDPathAndLineFromCallsites (callsites) {
+  if (callsites) {
+    for (let i = 0; i < callsites.length; i++) {
+      const callsite = callsites[i]
+      const path = callsite.getFileName()
+      if (!isExcluded(callsite) && path.indexOf(pathLine.ddBasePath) === -1) {
+        return {
+          path,
+          line: callsite.getLineNumber()
+        }
+      }
+    }
+  }
+  return null
+}
+
+function isExcluded (callsite) {
+  if (callsite.isNative()) return true
+  const filename = callsite.getFileName()
+  for (let i = 0; i < EXCLUDED_PATHS.length; i++) {
+    if (filename.indexOf(EXCLUDED_PATHS[i]) > -1) {
+      return true
+    }
+  }
+  for (let i = 0; i < EXCLUDED_PATH_PREFIXES.length; i++) {
+    if (filename.indexOf(EXCLUDED_PATH_PREFIXES[i]) === 0) {
+      return true
+    }
+  }
+  return false
+}
+
+function getFirstNonDDPathAndLine () {
+  return getFirstNonDDPathAndLineFromCallsites(getCallSiteInfo())
+}
+module.exports = pathLine