dd-trace 3.1.0 → 3.3.0

package/packages/dd-trace/src/exporters/span-stats/writer.js

@@ -0,0 +1,54 @@
+
+const { SpanStatsEncoder } = require('../../encode/span-stats')
+
+const pkg = require('../../../../../package.json')
+
+const BaseWriter = require('../common/writer')
+const request = require('../common/request')
+const log = require('../../log')
+
+class Writer extends BaseWriter {
+  constructor ({ url }) {
+    super(...arguments)
+    this._url = url
+    this._encoder = new SpanStatsEncoder(this)
+  }
+
+  _sendPayload (data, _, done) {
+    makeRequest(data, this._url, (err, res) => {
+      if (err) {
+        log.error(err)
+        done()
+        return
+      }
+      log.debug(`Response from the intake: ${res}`)
+      done()
+    })
+  }
+}
+
+function makeRequest (data, url, cb) {
+  const options = {
+    path: '/v0.6/stats',
+    method: 'PUT',
+    headers: {
+      'Datadog-Meta-Lang': 'javascript',
+      'Datadog-Meta-Tracer-Version': pkg.version,
+      'Content-Type': 'application/msgpack'
+    }
+  }
+
+  options.protocol = url.protocol
+  options.hostname = url.hostname
+  options.port = url.port
+
+  log.debug(() => `Request to the intake: ${JSON.stringify(options)}`)
+
+  request(data, options, (err, res) => {
+    cb(err, res)
+  })
+}
+
+module.exports = {
+  Writer
+}

package/packages/dd-trace/src/format.js

@@ -12,6 +12,7 @@ const SAMPLING_AGENT_DECISION = constants.SAMPLING_AGENT_DECISION
 const MEASURED = tags.MEASURED
 const ORIGIN_KEY = constants.ORIGIN_KEY
 const HOSTNAME_KEY = constants.HOSTNAME_KEY
+const TOP_LEVEL_KEY = constants.TOP_LEVEL_KEY
 
 const map = {
   'service.name': 'service',
@@ -110,6 +111,7 @@ function extractRootTags (trace, span) {
   addTag({}, trace.metrics, SAMPLING_RULE_DECISION, context._trace[SAMPLING_RULE_DECISION])
   addTag({}, trace.metrics, SAMPLING_LIMIT_DECISION, context._trace[SAMPLING_LIMIT_DECISION])
   addTag({}, trace.metrics, SAMPLING_AGENT_DECISION, context._trace[SAMPLING_AGENT_DECISION])
+  addTag({}, trace.metrics, TOP_LEVEL_KEY, 1)
 }
 
 function extractChunkTags (trace, span) {

package/packages/dd-trace/src/iitm.js

@@ -2,8 +2,19 @@
 
 const semver = require('semver')
 const logger = require('./log')
+const { addHook } = require('import-in-the-middle')
+const dc = require('diagnostics_channel')
 
 if (semver.satisfies(process.versions.node, '>=14.13.1')) {
+  const moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart')
+  addHook((name, namespace) => {
+    if (moduleLoadStartChannel.hasSubscribers) {
+      moduleLoadStartChannel.publish({
+        filename: name,
+        module: namespace
+      })
+    }
+  })
   module.exports = require('import-in-the-middle')
 } else {
   logger.warn('ESM is not fully supported by this version of Node.js, ' +

package/packages/dd-trace/src/opentracing/propagation/text_map.js

@@ -11,6 +11,7 @@ const traceKey = 'x-datadog-trace-id'
 const spanKey = 'x-datadog-parent-id'
 const originKey = 'x-datadog-origin'
 const samplingKey = 'x-datadog-sampling-priority'
+const tagsKey = 'x-datadog-tags'
 const baggagePrefix = 'ot-baggage-'
 const b3TraceKey = 'x-b3-traceid'
 const b3TraceExpr = /^([0-9a-f]{16}){1,2}$/i
@@ -23,6 +24,8 @@ const b3HeaderKey = 'b3'
 const sqsdHeaderHey = 'x-aws-sqsd-attr-_datadog'
 const b3HeaderExpr = /^(([0-9a-f]{16}){1,2}-[0-9a-f]{16}(-[01d](-[0-9a-f]{16})?)?|[01d])$/i
 const baggageExpr = new RegExp(`^${baggagePrefix}(.+)$`)
+const tagKeyExpr = /^_dd\.p\.[\x21-\x2b\x2d-\x7e]+$/ // ASCII minus spaces and commas
+const tagValueExpr = /^[\x20-\x2b\x2d-\x7e]*$/ // ASCII minus commas
 const ddKeys = [traceKey, spanKey, samplingKey, originKey]
 const b3Keys = [b3TraceKey, b3SpanKey, b3ParentKey, b3SampledKey, b3FlagsKey, b3HeaderKey]
 const logKeys = ddKeys.concat(b3Keys)
@@ -43,6 +46,7 @@ class TextMapPropagator {
     this._injectBaggageItems(spanContext, carrier)
     this._injectB3(spanContext, carrier)
     this._injectTraceparent(spanContext, carrier)
+    this._injectTags(spanContext, carrier)
 
     log.debug(() => `Inject into carrier: ${JSON.stringify(pick(carrier, logKeys))}.`)
   }
@@ -79,6 +83,35 @@ class TextMapPropagator {
     })
   }
 
+  _injectTags (spanContext, carrier) {
+    const trace = spanContext._trace
+
+    if (this._config.tagsHeaderMaxLength === 0) {
+      log.debug('Trace tag propagation is disabled, skipping injection.')
+      return
+    }
+
+    const tags = []
+
+    for (const key in trace.tags) {
+      if (!trace.tags[key] || !key.startsWith('_dd.p.')) continue
+      if (!this._validateTagKey(key) || !this._validateTagValue(trace.tags[key])) {
+        log.error('Trace tags from span are invalid, skipping injection.')
+        return
+      }
+
+      tags.push(`${key}=${trace.tags[key]}`)
+    }
+
+    const header = tags.join(',')
+
+    if (header.length > this._config.tagsHeaderMaxLength) {
+      log.error('Trace tags from span are too large, skipping injection.')
+    } else if (header) {
+      carrier[tagsKey] = header
+    }
+  }
+
   _injectB3 (spanContext, carrier) {
     if (!this._config.experimental.b3) return
 
@@ -118,6 +151,7 @@ class TextMapPropagator {
       this._extractOrigin(carrier, spanContext)
       this._extractBaggageItems(carrier, spanContext)
       this._extractSamplingPriority(carrier, spanContext)
+      this._extractTags(carrier, spanContext)
     }
 
     return spanContext
@@ -273,6 +307,43 @@ class TextMapPropagator {
     }
   }
 
+  _extractTags (carrier, spanContext) {
+    if (!carrier[tagsKey]) return
+
+    const trace = spanContext._trace
+
+    if (this._config.tagsHeaderMaxLength === 0) {
+      log.debug('Trace tag propagation is disabled, skipping extraction.')
+    } else if (carrier[tagsKey].length > this._config.tagsHeaderMaxLength) {
+      log.error('Trace tags from carrier are too large, skipping extraction.')
+    } else {
+      const pairs = carrier[tagsKey].split(',')
+      const tags = {}
+
+      for (const pair of pairs) {
+        const [key, ...rest] = pair.split('=')
+        const value = rest.join('=')
+
+        if (!this._validateTagKey(key) || !this._validateTagValue(value)) {
+          log.error('Trace tags from carrier are invalid, skipping extraction.')
+          return
+        }
+
+        tags[key] = value
+      }
+
+      Object.assign(trace.tags, tags)
+    }
+  }
+
+  _validateTagKey (key) {
+    return tagKeyExpr.test(key)
+  }
+
+  _validateTagValue (value) {
+    return tagValueExpr.test(value)
+  }
+
   _getPriority (sampled, debug) {
     if (debug) {
       return USER_KEEP

package/packages/dd-trace/src/opentracing/tracer.js

@@ -28,7 +28,7 @@ class DatadogTracer {
     this._tags = config.tags
     this._logInjection = config.logInjection
     this._debug = config.debug
-    this._prioritySampler = new PrioritySampler(config.env, config.experimental.sampler)
+    this._prioritySampler = new PrioritySampler(config.env, config.sampler)
     this._exporter = new Exporter(config, this._prioritySampler)
     this._processor = new SpanProcessor(this._exporter, this._prioritySampler, config)
     this._url = this._exporter._url

package/packages/dd-trace/src/plugin_manager.js

@@ -119,9 +119,11 @@ module.exports = class PluginManager {
     const {
       logInjection,
       serviceMapping,
-      experimental,
       queryStringObfuscation,
-      isIntelligentTestRunnerEnabled
+      clientIpHeaderDisabled,
+      clientIpHeader,
+      isIntelligentTestRunnerEnabled,
+      experimental
     } = this._tracerConfig
 
     const sharedConfig = {}
@@ -134,6 +136,14 @@ module.exports = class PluginManager {
       sharedConfig.queryStringObfuscation = queryStringObfuscation
     }
 
+    if (clientIpHeaderDisabled !== undefined) {
+      sharedConfig.clientIpHeaderDisabled = clientIpHeaderDisabled
+    }
+
+    if (clientIpHeader !== undefined) {
+      sharedConfig.clientIpHeader = clientIpHeader
+    }
+
     if (experimental) {
       sharedConfig.isAgentlessEnabled = experimental.exporter === 'datadog'
     }

package/packages/dd-trace/src/plugins/index.js

@@ -7,6 +7,7 @@ module.exports = {
   get '@google-cloud/pubsub' () { return require('../../../datadog-plugin-google-cloud-pubsub/src') },
   get '@grpc/grpc-js' () { return require('../../../datadog-plugin-grpc/src') },
   get '@hapi/hapi' () { return require('../../../datadog-plugin-hapi/src') },
+  get '@jest/core' () { return require('../../../datadog-plugin-jest/src') },
   get '@koa/router' () { return require('../../../datadog-plugin-koa/src') },
   get '@node-redis/client' () { return require('../../../datadog-plugin-redis/src') },
   get 'amqp10' () { return require('../../../datadog-plugin-amqp10/src') },
@@ -29,6 +30,8 @@ module.exports = {
   get 'http2' () { return require('../../../datadog-plugin-http2/src') },
   get 'https' () { return require('../../../datadog-plugin-http/src') },
   get 'ioredis' () { return require('../../../datadog-plugin-ioredis/src') },
+  get 'jest-circus' () { return require('../../../datadog-plugin-jest/src') },
+  get 'jest-config' () { return require('../../../datadog-plugin-jest/src') },
   get 'jest-environment-node' () { return require('../../../datadog-plugin-jest/src') },
   get 'jest-environment-jsdom' () { return require('../../../datadog-plugin-jest/src') },
   get 'jest-jasmine2' () { return require('../../../datadog-plugin-jest/src') },

package/packages/dd-trace/src/plugins/log_plugin.js

@@ -9,25 +9,32 @@ const hasOwn = (obj, prop) => Object.prototype.hasOwnProperty.call(obj, prop)
 function messageProxy (message, holder) {
   return new Proxy(message, {
     get (target, p, receiver) {
-      switch (p) {
-        case Symbol.toStringTag:
-          return Object.prototype.toString.call(target).slice(8, -1)
-        case 'dd':
-          return holder.dd
-        default:
-          return Reflect.get(target, p, receiver)
+      if (p === Symbol.toStringTag) {
+        return Object.prototype.toString.call(target).slice(8, -1)
       }
+
+      if (shouldOverride(target, p)) {
+        return holder.dd
+      }
+
+      return Reflect.get(target, p, receiver)
     },
     ownKeys (target) {
       const ownKeys = Reflect.ownKeys(target)
-      return hasOwn(target, 'dd') ? ownKeys : ['dd', ...ownKeys]
+      return hasOwn(target, 'dd') || !Reflect.isExtensible(target)
+        ? ownKeys
+        : ['dd', ...ownKeys]
     },
     getOwnPropertyDescriptor (target, p) {
-      return Reflect.getOwnPropertyDescriptor(p === 'dd' ? holder : target, p)
+      return Reflect.getOwnPropertyDescriptor(shouldOverride(target, p) ? holder : target, p)
     }
   })
 }
 
+function shouldOverride (target, p) {
+  return p === 'dd' && !Reflect.has(target, p) && Reflect.isExtensible(target)
+}
+
 module.exports = class LogPlugin extends Plugin {
   constructor (...args) {
     super(...args)

package/packages/dd-trace/src/plugins/util/ip_blocklist.js

@@ -0,0 +1,51 @@
+'use strict'
+
+const semver = require('semver')
+
+if (semver.satisfies(process.version, '>=14.18.0')) {
+  const net = require('net')
+
+  module.exports = net.BlockList
+} else {
+  const ipaddr = require('ipaddr.js')
+
+  module.exports = class BlockList {
+    constructor () {
+      this.v4Ranges = []
+      this.v6Ranges = []
+    }
+
+    addSubnet (net, prefix, type) {
+      this[type === 'ipv4' ? 'v4Ranges' : 'v6Ranges'].push(ipaddr.parseCIDR(`${net}/${prefix}`))
+    }
+
+    check (address, type) {
+      try {
+        let ip = ipaddr.parse(address)
+
+        type = ip.kind()
+
+        if (type === 'ipv6') {
+          for (const range of this.v6Ranges) {
+            if (ip.match(range)) return true
+          }
+
+          if (ip.isIPv4MappedAddress()) {
+            ip = ip.toIPv4Address()
+            type = ip.kind()
+          }
+        }
+
+        if (type === 'ipv4') {
+          for (const range of this.v4Ranges) {
+            if (ip.match(range)) return true
+          }
+        }
+
+        return false
+      } catch {
+        return false
+      }
+    }
+  }
+}

package/packages/dd-trace/src/plugins/util/web.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const net = require('net')
 const uniq = require('lodash.uniq')
 const analyticsSampler = require('../../analytics_sampler')
 const FORMAT_HTTP_HEADERS = 'http_headers'
@@ -8,6 +9,7 @@ const tags = require('../../../../../ext/tags')
 const types = require('../../../../../ext/types')
 const kinds = require('../../../../../ext/kinds')
 const urlFilter = require('./urlfilter')
+const BlockList = require('./ip_blocklist')
 const { incomingHttpRequestEnd } = require('../../appsec/gateway/channels')
 
 const WEB = types.WEB
@@ -24,12 +26,25 @@ const HTTP_ROUTE = tags.HTTP_ROUTE
 const HTTP_REQUEST_HEADERS = tags.HTTP_REQUEST_HEADERS
 const HTTP_RESPONSE_HEADERS = tags.HTTP_RESPONSE_HEADERS
 const HTTP_USERAGENT = tags.HTTP_USERAGENT
+const HTTP_CLIENT_IP = tags.HTTP_CLIENT_IP
 const MANUAL_DROP = tags.MANUAL_DROP
 
 const HTTP2_HEADER_AUTHORITY = ':authority'
 const HTTP2_HEADER_SCHEME = ':scheme'
 const HTTP2_HEADER_PATH = ':path'
 
+const ipHeaderList = [
+  'x-forwarded-for',
+  'x-real-ip',
+  'client-ip',
+  'x-forwarded',
+  'x-cluster-client-ip',
+  'forwarded-for',
+  'forwarded',
+  'via',
+  'true-client-ip'
+]
+
 const contexts = new WeakMap()
 const ends = new WeakMap()
 
@@ -337,6 +352,42 @@ const web = {
 
     return `${path}?${qs}`
   },
+
+  extractIp (context) {
+    const { req, config } = context
+
+    if (config.clientIpHeaderDisabled) return
+
+    const headers = req.headers
+
+    if (config.clientIpHeader) {
+      const header = headers[config.clientIpHeader]
+      if (!header) return
+
+      return findFirstIp(header)
+    }
+
+    const foundHeaders = []
+
+    for (let i = 0; i < ipHeaderList.length; i++) {
+      if (headers[ipHeaderList[i]]) {
+        foundHeaders.push(ipHeaderList[i])
+      }
+    }
+
+    if (foundHeaders.length === 1) {
+      const header = headers[foundHeaders[0]]
+      const firstIp = findFirstIp(header)
+
+      if (firstIp) return firstIp
+    } else if (foundHeaders.length > 1) {
+      log.error(`Cannot find client IP: multiple IP headers detected ${foundHeaders}`)
+      return
+    }
+
+    return req.socket && req.socket.remoteAddress
+  },
+
   wrapWriteHead (context) {
     const { req, res } = context
     const writeHead = res.writeHead
@@ -392,7 +443,8 @@ function addAllowHeaders (req, res, headers) {
     'x-datadog-parent-id',
     'x-datadog-sampled', // Deprecated, but still accept it in case it's sent.
     'x-datadog-sampling-priority',
-    'x-datadog-trace-id'
+    'x-datadog-trace-id',
+    'x-datadog-tags'
   ]
 
   for (const header of contextHeaders) {
@@ -434,7 +486,8 @@ function addRequestTags (context) {
     [HTTP_METHOD]: req.method,
     [SPAN_KIND]: SERVER,
     [SPAN_TYPE]: WEB,
-    [HTTP_USERAGENT]: req.headers['user-agent']
+    [HTTP_USERAGENT]: req.headers['user-agent'],
+    [HTTP_CLIENT_IP]: web.extractIp(context)
   })
 
   addHeaders(context)
@@ -502,6 +555,51 @@ function getProtocol (req) {
   return 'http'
 }
 
+const privateCIDRs = [
+  '127.0.0.0/8',
+  '10.0.0.0/8',
+  '172.16.0.0/12',
+  '192.168.0.0/16',
+  '169.254.0.0/16',
+  '::1/128',
+  'fec0::/10',
+  'fe80::/10',
+  'fc00::/7',
+  'fd00::/8'
+]
+
+const privateIPMatcher = new BlockList()
+
+for (const cidr of privateCIDRs) {
+  const [ address, prefix ] = cidr.split('/')
+
+  privateIPMatcher.addSubnet(address, parseInt(prefix), net.isIPv6(address) ? 'ipv6' : 'ipv4')
+}
+
+function findFirstIp (str) {
+  let firstPrivateIp
+  const splitted = str.split(',')
+
+  for (let i = 0; i < splitted.length; i++) {
+    const chunk = splitted[i].trim()
+
+    // TODO: strip port and interface data ?
+
+    const type = net.isIP(chunk)
+    if (!type) continue
+
+    if (!privateIPMatcher.check(chunk, type === 6 ? 'ipv6' : 'ipv4')) {
+      // it's public, return it immediately
+      return chunk
+    }
+
+    // it's private, only save the first one found
+    if (!firstPrivateIp) firstPrivateIp = chunk
+  }
+
+  return firstPrivateIp
+}
+
 function getHeadersToRecord (config) {
   if (Array.isArray(config.headers)) {
     try {

package/packages/dd-trace/src/priority_sampler.js

@@ -6,9 +6,14 @@ const ext = require('../../../ext')
 const { setSamplingRules } = require('./startup-log')
 
 const {
+  SAMPLING_MECHANISM_DEFAULT,
+  SAMPLING_MECHANISM_AGENT,
+  SAMPLING_MECHANISM_RULE,
+  SAMPLING_MECHANISM_MANUAL,
   SAMPLING_RULE_DECISION,
   SAMPLING_LIMIT_DECISION,
-  SAMPLING_AGENT_DECISION
+  SAMPLING_AGENT_DECISION,
+  DECISION_MAKER_KEY
 } = require('./constants')
 
 const SERVICE_NAME = ext.tags.SERVICE_NAME
@@ -45,6 +50,7 @@ class PrioritySampler {
     const context = this._getContext(span)
     const root = context._trace.started[0]
 
+    // TODO: remove the decision maker tag when priority is less than AUTO_KEEP
     if (context._sampling.priority !== undefined) return
     if (!root) return // noop span
 
@@ -52,9 +58,14 @@ class PrioritySampler {
 
     if (this.validate(tag)) {
       context._sampling.priority = tag
+      context._sampling.mechanism = SAMPLING_MECHANISM_MANUAL
     } else if (auto) {
       context._sampling.priority = this._getPriorityFromAuto(root)
+    } else {
+      return
     }
+
+    this._addDecisionMaker(root)
   }
 
   update (rates) {
@@ -115,6 +126,7 @@ class PrioritySampler {
 
   _getPriorityByRule (context, rule) {
     context._trace[SAMPLING_RULE_DECISION] = rule.sampleRate
+    context._sampling.mechanism = SAMPLING_MECHANISM_RULE
 
     return rule.sampler.isSampled(context) && this._isSampledByRateLimit(context) ? USER_KEEP : USER_REJECT
   }
@@ -133,10 +145,33 @@ class PrioritySampler {
 
     context._trace[SAMPLING_AGENT_DECISION] = sampler.rate()
 
+    if (sampler === defaultSampler) {
+      context._sampling.mechanism = SAMPLING_MECHANISM_DEFAULT
+    } else {
+      context._sampling.mechanism = SAMPLING_MECHANISM_AGENT
+    }
+
     return sampler.isSampled(context) ? AUTO_KEEP : AUTO_REJECT
   }
 
+  _addDecisionMaker (span) {
+    const context = span.context()
+    const trace = context._trace
+    const priority = context._sampling.priority
+    const mechanism = context._sampling.mechanism
+
+    if (priority >= AUTO_KEEP) {
+      if (!trace.tags[DECISION_MAKER_KEY]) {
+        trace.tags[DECISION_MAKER_KEY] = `-${mechanism}`
+      }
+    } else {
+      delete trace.tags[DECISION_MAKER_KEY]
+    }
+  }
+
   _normalizeRules (rules, sampleRate) {
+    rules = [].concat(rules || [])
+
     return rules
       .concat({ sampleRate })
       .map(rule => ({ ...rule, sampleRate: parseFloat(rule.sampleRate) }))

package/packages/dd-trace/src/proxy.js

@@ -48,6 +48,9 @@ class Tracer extends NoopProxy {
         if (config.appsec.enabled) {
           require('./appsec').enable(config)
         }
+        if (config.iast.enabled) {
+          require('./appsec/iast').enable(config)
+        }
 
         this._tracer = new DatadogTracer(config)
         this._pluginManager.configure(config)

package/packages/dd-trace/src/ritm.js

@@ -3,6 +3,7 @@
 const path = require('path')
 const Module = require('module')
 const parse = require('module-details-from-path')
+const dc = require('diagnostics_channel')
 
 const origRequire = Module.prototype.require
 
@@ -14,7 +15,7 @@ let moduleHooks = Object.create(null)
 let cache = Object.create(null)
 let patching = Object.create(null)
 let patchedRequire = null
-
+const moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart')
 function Hook (modules, options, onrequire) {
   if (!(this instanceof Hook)) return new Hook(modules, options, onrequire)
   if (typeof modules === 'function') {
@@ -78,6 +79,14 @@ function Hook (modules, options, onrequire) {
     // so the patching mark can be cleaned up.
     delete patching[filename]
 
+    if (moduleLoadStartChannel.hasSubscribers) {
+      moduleLoadStartChannel.publish({
+        filename,
+        module: exports,
+        request
+      })
+    }
+
     if (core) {
       hooks = moduleHooks[filename]
       if (!hooks) return exports // abort if module name isn't on whitelist

package/packages/dd-trace/src/span_processor.js

@@ -3,6 +3,8 @@
 const log = require('./log')
 const format = require('./format')
 
+const { SpanStatsProcessor } = require('./span_stats')
+
 const startedSpans = new WeakSet()
 const finishedSpans = new WeakSet()
 
@@ -11,6 +13,8 @@ class SpanProcessor {
     this._exporter = exporter
     this._prioritySampler = prioritySampler
     this._config = config
+
+    this._stats = new SpanStatsProcessor(config)
   }
 
   process (span) {
@@ -26,7 +30,9 @@ class SpanProcessor {
 
       for (const span of started) {
         if (span._duration !== undefined) {
-          formatted.push(format(span))
+          const formattedSpan = format(span)
+          this._stats.onSpanFinished(formattedSpan)
+          formatted.push(formattedSpan)
         } else {
           active.push(span)
         }