dd-trace 2.17.0 → 2.19.0

package/packages/datadog-plugin-sharedb/src/index.js

@@ -1,48 +1,31 @@
 'use strict'
 
-const Plugin = require('../../dd-trace/src/plugins/plugin')
-const { storage } = require('../../datadog-core')
-
-class SharedbPlugin extends Plugin {
-  static get name () {
-    return 'sharedb'
-  }
-
-  constructor (...args) {
-    super(...args)
-
-    this.addSub(`apm:sharedb:request:start`, ({ actionName, request }) => {
-      const store = storage.getStore()
-      const childOf = store ? store.span : store
-      const span = this.tracer.startSpan('sharedb.request', {
-        childOf,
-        tags: {
-          'service.name': this.config.service || this.tracer._service,
-          'span.kind': 'server',
-          'sharedb.action': actionName,
-          'resource.name': getReadableResourceName(actionName, request.c, request.q)
-        }
-      })
-
-      if (this.config.hooks && this.config.hooks.receive) {
-        this.config.hooks.receive(span, request)
+const ServerPlugin = require('../../dd-trace/src/plugins/server')
+
+class SharedbPlugin extends ServerPlugin {
+  static get name () { return 'sharedb' }
+
+  start ({ actionName, request }) {
+    const span = this.startSpan('sharedb.request', {
+      service: this.config.service,
+      resource: getReadableResourceName(actionName, request.c, request.q),
+      kind: 'server',
+      meta: {
+        'sharedb.action': actionName
       }
-
-      this.enter(span, store)
     })
 
-    this.addSub(`apm:sharedb:request:error`, err => {
-      const span = storage.getStore().span
-      span.setTag('error', err)
-    })
+    if (this.config.hooks && this.config.hooks.receive) {
+      this.config.hooks.receive(span, request)
+    }
+  }
 
-    this.addSub(`apm:sharedb:request:finish`, ({ request, res }) => {
-      const span = storage.getStore().span
-      if (this.config.hooks && this.config.hooks.reply) {
-        this.config.hooks.reply(span, request, res)
-      }
-      span.finish()
-    })
+  finish ({ request, res }) {
+    const span = this.activeSpan
+    if (this.config.hooks && this.config.hooks.reply) {
+      this.config.hooks.reply(span, request, res)
+    }
+    span.finish()
   }
 }
 

package/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js

@@ -2,7 +2,7 @@
 
 const Plugin = require('../../../../src/plugins/plugin')
 const { storage } = require('../../../../../datadog-core')
-const { getFirstNonDDPathAndLine } = require('./../path-line')
+const { getFirstNonDDPathAndLine } = require('../path-line')
 const { createVulnerability, addVulnerability } = require('../vulnerability-reporter')
 const { getIastContext } = require('../iast-context')
 const overheadController = require('../overhead-controller')

package/packages/dd-trace/src/appsec/iast/index.js

@@ -4,8 +4,7 @@ const web = require('../../plugins/util/web')
 const { storage } = require('../../../../datadog-core')
 const overheadController = require('./overhead-controller')
 const dc = require('diagnostics_channel')
-const { saveIastContext, getIastContext, cleanIastContext } = require('./iast-context')
-
+const iastContextFunctions = require('./iast-context')
 // TODO Change to `apm:http:server:request:[start|close]` when the subscription
 //  order of the callbacks can be enforce
 const requestStart = dc.channel('dd-trace:incomingHttpRequestStart')
@@ -33,7 +32,7 @@ function onIncomingHttpRequestStart (data) {
         const rootSpan = topContext.span
         const isRequestAcquired = overheadController.acquireRequest(rootSpan)
         if (isRequestAcquired) {
-          const iastContext = saveIastContext(store, topContext, { rootSpan, req: data.req })
+          const iastContext = iastContextFunctions.saveIastContext(store, topContext, { rootSpan, req: data.req })
           overheadController.initializeRequestContext(iastContext)
         }
       }
@@ -44,13 +43,12 @@ function onIncomingHttpRequestStart (data) {
 function onIncomingHttpRequestEnd (data) {
   if (data && data.req) {
     const store = storage.getStore()
-    const iastContext = getIastContext(storage.getStore())
+    const iastContext = iastContextFunctions.getIastContext(storage.getStore())
     if (iastContext && iastContext.rootSpan) {
-      overheadController.releaseRequest()
       sendVulnerabilities(iastContext, iastContext.rootSpan)
     }
     // TODO web.getContext(data.req) is required when the request is aborted
-    if (cleanIastContext(store, web.getContext(data.req), iastContext)) {
+    if (iastContextFunctions.cleanIastContext(store, web.getContext(data.req), iastContext)) {
       overheadController.releaseRequest()
     }
   }

package/packages/dd-trace/src/appsec/iast/path-line.js

@@ -51,6 +51,9 @@ function getFirstNonDDPathAndLineFromCallsites (callsites) {
 function isExcluded (callsite) {
   if (callsite.isNative()) return true
   const filename = callsite.getFileName()
+  if (!filename) {
+    return true
+  }
   for (let i = 0; i < EXCLUDED_PATHS.length; i++) {
     if (filename.indexOf(EXCLUDED_PATHS[i]) > -1) {
       return true

package/packages/dd-trace/src/appsec/recommended.json

@@ -1,9 +1,34 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.4.0"
+    "rules_version": "1.4.2"
   },
   "rules": [
+    {
+      "id": "blk-001-001",
+      "name": "Block IP Addresses",
+      "tags": {
+        "type": "block_ip",
+        "category": "security_response"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "http.client_ip"
+              }
+            ],
+            "data": "blocked_ips"
+          },
+          "operator": "ip_match"
+        }
+      ],
+      "transformers": [],
+      "on_match": [
+        "block"
+      ]
+    },
     {
       "id": "crs-913-110",
       "name": "Acunetix",
@@ -2828,51 +2853,6 @@
       ],
       "transformers": []
     },
-    {
-      "id": "crs-941-100",
-      "name": "XSS Attack Detected via libinjection",
-      "tags": {
-        "type": "xss",
-        "crs_id": "941100",
-        "category": "attack_attempt"
-      },
-      "conditions": [
-        {
-          "parameters": {
-            "inputs": [
-              {
-                "address": "server.request.headers.no_cookies",
-                "key_path": [
-                  "user-agent"
-                ]
-              },
-              {
-                "address": "server.request.headers.no_cookies",
-                "key_path": [
-                  "referer"
-                ]
-              },
-              {
-                "address": "server.request.query"
-              },
-              {
-                "address": "server.request.body"
-              },
-              {
-                "address": "server.request.path_params"
-              },
-              {
-                "address": "grpc.server.request.message"
-              }
-            ]
-          },
-          "operator": "is_xss"
-        }
-      ],
-      "transformers": [
-        "removeNulls"
-      ]
-    },
     {
       "id": "crs-941-110",
       "name": "XSS Filter - Category 1: Script Tag Vector",
@@ -4338,6 +4318,40 @@
         "keys_only"
       ]
     },
+    {
+      "id": "dog-000-007",
+      "name": "Server side template injection: Velocity & Freemarker",
+      "tags": {
+        "type": "java_code_injection",
+        "category": "attack_attempt"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              },
+              {
+                "address": "server.request.headers.no_cookies"
+              },
+              {
+                "address": "grpc.server.request.message"
+              }
+            ],
+            "regex": "#(?:set|foreach|macro|parse|if)\\(.*\\)|<#assign.*>"
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": []
+    },
     {
       "id": "nfd-000-001",
       "name": "Detect common directory discovery scans",

package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js

@@ -26,13 +26,10 @@ class Writer extends BaseWriter {
         'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY,
         ...form.getHeaders()
       },
-      timeout: 15000
+      timeout: 15000,
+      url: this._url
     }
 
-    options.protocol = this._url.protocol
-    options.hostname = this._url.hostname
-    options.port = this._url.port
-
     log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
 
     request(form, options, (err, res) => {

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -27,13 +27,10 @@ class Writer extends BaseWriter {
         'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY,
         'Content-Type': 'application/msgpack'
       },
-      timeout: 15000
+      timeout: 15000,
+      url: this._url
     }
 
-    options.protocol = this._url.protocol
-    options.hostname = this._url.hostname
-    options.port = this._url.port
-
     log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
     request(data, options, (err, res) => {
       if (err) {

package/packages/dd-trace/src/config.js

@@ -13,6 +13,15 @@ const uuid = require('crypto-randomuuid')
 const fromEntries = Object.fromEntries || (entries =>
   entries.reduce((obj, [k, v]) => Object.assign(obj, { [k]: v }), {}))
 
+function maybeFile (filepath) {
+  if (!filepath) return
+  try {
+    return fs.readFileSync(filepath, 'utf8')
+  } catch (e) {
+    return undefined
+  }
+}
+
 function safeJsonParse (input) {
   try {
     return JSON.parse(input)
@@ -133,14 +142,14 @@ class Config {
       parseInt(process.env.DD_TRACE_PARTIAL_FLUSH_MIN_SPANS),
       1000
     )
-    const DD_TRACE_CLIENT_IP_HEADER_DISABLED = coalesce(
-      process.env.DD_TRACE_CLIENT_IP_HEADER_DISABLED,
-      false
-    )
     const DD_TRACE_CLIENT_IP_HEADER = coalesce(
       process.env.DD_TRACE_CLIENT_IP_HEADER,
       null
     )
+    const DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP = coalesce(
+      process.env.DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP,
+      '.*'
+    )
     const DD_TRACE_B3_ENABLED = coalesce(
       options.experimental && options.experimental.b3,
       process.env.DD_TRACE_EXPERIMENTAL_B3_ENABLED,
@@ -260,10 +269,25 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
         ingestion.sampleRate
       ),
       rateLimit: coalesce(options.rateLimit, process.env.DD_TRACE_RATE_LIMIT, ingestion.rateLimit),
-      rules: coalesce(options.samplingRules, safeJsonParse(process.env.DD_TRACE_SAMPLING_RULES), []).map(rule => {
+      rules: coalesce(
+        options.samplingRules,
+        safeJsonParse(process.env.DD_TRACE_SAMPLING_RULES),
+        []
+      ).map(rule => {
         return remapify(rule, {
           sample_rate: 'sampleRate'
         })
+      }),
+      spanSamplingRules: coalesce(
+        options.spanSamplingRules,
+        safeJsonParse(maybeFile(process.env.DD_SPAN_SAMPLING_RULES_FILE)),
+        safeJsonParse(process.env.DD_SPAN_SAMPLING_RULES),
+        []
+      ).map(rule => {
+        return remapify(rule, {
+          sample_rate: 'sampleRate',
+          max_per_second: 'maxPerSecond'
+        })
       })
     }
 
@@ -282,8 +306,9 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     this.flushInterval = coalesce(parseInt(options.flushInterval, 10), defaultFlushInterval)
     this.flushMinSpans = DD_TRACE_PARTIAL_FLUSH_MIN_SPANS
     this.sampleRate = coalesce(Math.min(Math.max(sampler.sampleRate, 0), 1), 1)
-    this.clientIpHeaderDisabled = isTrue(DD_TRACE_CLIENT_IP_HEADER_DISABLED)
+    this.clientIpHeaderDisabled = !isTrue(DD_APPSEC_ENABLED)
     this.clientIpHeader = DD_TRACE_CLIENT_IP_HEADER
+    this.queryStringObfuscation = DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP
     this.logger = options.logger
     this.plugins = !!coalesce(options.plugins, true)
     this.service = DD_SERVICE

package/packages/dd-trace/src/constants.js

@@ -15,6 +15,9 @@ module.exports = {
   SAMPLING_MECHANISM_MANUAL: 4,
   SAMPLING_MECHANISM_APPSEC: 5,
   SAMPLING_MECHANISM_SPAN: 8,
+  SPAN_SAMPLING_MECHANISM: '_dd.span_sampling.mechanism',
+  SPAN_SAMPLING_RULE_RATE: '_dd.span_sampling.rule_rate',
+  SPAN_SAMPLING_MAX_PER_SECOND: '_dd.span_sampling.max_per_second',
   DATADOG_LAMBDA_EXTENSION_PATH: '/opt/extensions/datadog-agent',
   DECISION_MAKER_KEY: '_dd.p.dm'
 }

package/packages/dd-trace/src/dogstatsd.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const lookup = require('dns').lookup // cache to avoid instrumentation
+const request = require('./exporters/common/request')
 const dgram = require('dgram')
 const isIP = require('net').isIP
 const log = require('./log')
@@ -11,6 +12,13 @@ class Client {
   constructor (options) {
     options = options || {}
 
+    if (options.metricsProxyUrl) {
+      this._httpOptions = {
+        url: options.metricsProxyUrl.toString(),
+        path: '/dogstatsd/v2/proxy'
+      }
+    }
+
     this._host = options.host || 'localhost'
     this._family = isIP(this._host)
     this._port = options.port || 8125
@@ -38,26 +46,50 @@ class Client {
 
     this._queue = []
 
+    if (this._httpOptions) {
+      this._sendHttp(queue)
+    } else {
+      this._sendUdp(queue)
+    }
+  }
+
+  _sendHttp (queue) {
+    const buffer = Buffer.concat(queue)
+    request(buffer, this._httpOptions, (err) => {
+      if (err) {
+        log.error('HTTP error from agent: ' + err.stack)
+        if (err.status) {
+          // Inside this if-block, we have connectivity to the agent, but
+          // we're not getting a 200 from the proxy endpoint. If it's a 404,
+          // then we know we'll never have the endpoint, so just clear out the
+          // options. Either way, we can give UDP a try.
+          if (err.status === 404) {
+            this._httpOptions = null
+          }
+          this._sendUdp(queue)
+        }
+      }
+    })
+  }
+
+  _sendUdp (queue) {
     if (this._family !== 0) {
-      this._sendAll(queue, this._host, this._family)
+      this._sendUdpFromQueue(queue, this._host, this._family)
     } else {
       lookup(this._host, (err, address, family) => {
         if (err) return log.error(err)
-        this._sendAll(queue, address, family)
+        this._sendUdpFromQueue(queue, address, family)
       })
     }
   }
 
-  _send (address, family, buffer) {
+  _sendUdpFromQueue (queue, address, family) {
     const socket = family === 6 ? this._udp6 : this._udp4
 
-    log.debug(`Sending to DogStatsD: ${buffer}`)
-
-    socket.send(buffer, 0, buffer.length, this._port, address)
-  }
-
-  _sendAll (queue, address, family) {
-    queue.forEach((buffer) => this._send(address, family, buffer))
+    queue.forEach((buffer) => {
+      log.debug(`Sending to DogStatsD: ${buffer}`)
+      socket.send(buffer, 0, buffer.length, this._port, address)
+    })
   }
 
   _add (stat, value, type, tags) {

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -197,8 +197,16 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
   }
 
   _encode (bytes, trace) {
-    this._eventCount += trace.length
-    const events = trace.map(formatSpan)
+    const rawEvents = trace.map(formatSpan)
+
+    const testSessionEvents = rawEvents.filter(
+      event => event.type === 'test_session_end' || event.type === 'test_suite_end'
+    )
+
+    const isTestSessionTrace = !!testSessionEvents.length
+    const events = isTestSessionTrace ? testSessionEvents : rawEvents
+
+    this._eventCount += events.length
 
     for (const event of events) {
       this._encodeEvent(bytes, event)

package/packages/dd-trace/src/exporters/agent/writer.js

@@ -85,21 +85,14 @@ function makeRequest (version, data, count, url, headers, lookup, needsStartupLo
       'Datadog-Meta-Tracer-Version': tracerVersion,
       'X-Datadog-Trace-Count': String(count)
     },
-    lookup
+    lookup,
+    url
   }
 
   setHeader(options.headers, 'Datadog-Meta-Lang', 'nodejs')
   setHeader(options.headers, 'Datadog-Meta-Lang-Version', process.version)
   setHeader(options.headers, 'Datadog-Meta-Lang-Interpreter', process.jsEngine || 'v8')
 
-  if (url.protocol === 'unix:') {
-    options.socketPath = url.pathname
-  } else {
-    options.protocol = url.protocol
-    options.hostname = url.hostname
-    options.port = url.port
-  }
-
   log.debug(() => `Request to the agent: ${JSON.stringify(options)}`)
 
   request(data, options, (err, res, status) => {

package/packages/dd-trace/src/exporters/common/request.js

@@ -6,6 +6,7 @@
 const { Readable } = require('stream')
 const http = require('http')
 const https = require('https')
+const { parse: urlParse } = require('url')
 const docker = require('./docker')
 const { storage } = require('../../../../datadog-core')
 const log = require('../../log')
@@ -24,6 +25,17 @@ function request (data, options, callback) {
     options.headers = {}
   }
 
+  if (options.url) {
+    const url = typeof options.url === 'object' ? options.url : urlParse(options.url)
+    if (url.protocol === 'unix:') {
+      options.socketPath = url.pathname
+    } else {
+      options.protocol = url.protocol
+      options.hostname = url.hostname
+      options.port = url.port
+    }
+  }
+
   const isReadable = data instanceof Readable
 
   // The timeout should be kept low to avoid excessive queueing.

package/packages/dd-trace/src/format.js

@@ -9,6 +9,10 @@ const SAMPLING_PRIORITY_KEY = constants.SAMPLING_PRIORITY_KEY
 const SAMPLING_RULE_DECISION = constants.SAMPLING_RULE_DECISION
 const SAMPLING_LIMIT_DECISION = constants.SAMPLING_LIMIT_DECISION
 const SAMPLING_AGENT_DECISION = constants.SAMPLING_AGENT_DECISION
+const SPAN_SAMPLING_MECHANISM = constants.SPAN_SAMPLING_MECHANISM
+const SPAN_SAMPLING_RULE_RATE = constants.SPAN_SAMPLING_RULE_RATE
+const SPAN_SAMPLING_MAX_PER_SECOND = constants.SPAN_SAMPLING_MAX_PER_SECOND
+const SAMPLING_MECHANISM_SPAN = constants.SAMPLING_MECHANISM_SPAN
 const MEASURED = tags.MEASURED
 const ORIGIN_KEY = constants.ORIGIN_KEY
 const HOSTNAME_KEY = constants.HOSTNAME_KEY
@@ -47,6 +51,13 @@ function formatSpan (span) {
   }
 }
 
+function setSingleSpanIngestionTags (span, options) {
+  if (!options) return
+  addTag({}, span.metrics, SPAN_SAMPLING_MECHANISM, SAMPLING_MECHANISM_SPAN)
+  addTag({}, span.metrics, SPAN_SAMPLING_RULE_RATE, options.sampleRate)
+  addTag({}, span.metrics, SPAN_SAMPLING_MAX_PER_SECOND, options.maxPerSecond)
+}
+
 function extractTags (trace, span) {
   const context = span.context()
   const origin = context._trace.origin
@@ -96,6 +107,8 @@ function extractTags (trace, span) {
     addTag(trace.meta, trace.metrics, 'language', 'javascript')
   }
 
+  setSingleSpanIngestionTags(trace, context._sampling.spanSampling)
+
   addTag(trace.meta, trace.metrics, SAMPLING_PRIORITY_KEY, priority)
   addTag(trace.meta, trace.metrics, ORIGIN_KEY, origin)
   addTag(trace.meta, trace.metrics, HOSTNAME_KEY, hostname)

package/packages/dd-trace/src/metrics.js

@@ -48,11 +48,19 @@ module.exports = {
       nativeMetrics = null
     }
 
-    client = new Client({
+    const clientConfig = {
       host: config.dogstatsd.hostname,
       port: config.dogstatsd.port,
       tags
-    })
+    }
+
+    if (config.url) {
+      clientConfig.metricsProxyUrl = config.url
+    } else if (config.port) {
+      clientConfig.metricsProxyUrl = new URL(`http://${config.hostname || 'localhost'}:${config.port}`)
+    }
+
+    client = new Client(clientConfig)
 
     time = process.hrtime()
 

package/packages/dd-trace/src/plugin_manager.js

@@ -123,7 +123,8 @@ module.exports = class PluginManager {
       clientIpHeader,
       isIntelligentTestRunnerEnabled,
       site,
-      experimental
+      experimental,
+      queryStringObfuscation
     } = this._tracerConfig
 
     const sharedConfig = {}
@@ -132,6 +133,10 @@ module.exports = class PluginManager {
       sharedConfig.logInjection = logInjection
     }
 
+    if (queryStringObfuscation !== undefined) {
+      sharedConfig.queryStringObfuscation = queryStringObfuscation
+    }
+
     if (clientIpHeaderDisabled !== undefined) {
       sharedConfig.clientIpHeaderDisabled = clientIpHeaderDisabled
     }

package/packages/dd-trace/src/plugins/client.js

@@ -2,6 +2,8 @@
 
 const OutgoingPlugin = require('./outgoing')
 
-class ClientPlugin extends OutgoingPlugin {}
+class ClientPlugin extends OutgoingPlugin {
+  static get operation () { return 'request' }
+}
 
 module.exports = ClientPlugin

package/packages/dd-trace/src/plugins/composite.js

@@ -0,0 +1,26 @@
+'use strict'
+
+const Plugin = require('./plugin')
+
+class CompositePlugin extends Plugin {
+  constructor (...args) {
+    super(...args)
+
+    for (const [name, PluginClass] of Object.entries(this.constructor.plugins)) {
+      this[name] = new PluginClass(...args)
+    }
+  }
+
+  configure (config) {
+    for (const name in this.constructor.plugins) {
+      const pluginConfig = config[name] === false ? false : {
+        ...config,
+        ...config[name]
+      }
+
+      this[name].configure(pluginConfig)
+    }
+  }
+}
+
+module.exports = CompositePlugin

package/packages/dd-trace/src/plugins/consumer.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const IncomingPlugin = require('./incoming')
+
+class ConsumerPlugin extends IncomingPlugin {
+  static get operation () { return 'receive' }
+}
+
+module.exports = ConsumerPlugin

package/packages/dd-trace/src/plugins/incoming.js

@@ -0,0 +1,7 @@
+'use strict'
+
+const TracingPlugin = require('./tracing')
+
+class IncomingPlugin extends TracingPlugin {}
+
+module.exports = IncomingPlugin

package/packages/dd-trace/src/plugins/index.js

@@ -10,6 +10,7 @@ module.exports = {
   get '@jest/core' () { return require('../../../datadog-plugin-jest/src') },
   get '@koa/router' () { return require('../../../datadog-plugin-koa/src') },
   get '@node-redis/client' () { return require('../../../datadog-plugin-redis/src') },
+  get '@opensearch-project/opensearch' () { return require('../../../datadog-plugin-opensearch/src') },
   get '@redis/client' () { return require('../../../datadog-plugin-redis/src') },
   get 'amqp10' () { return require('../../../datadog-plugin-amqp10/src') },
   get 'amqplib' () { return require('../../../datadog-plugin-amqplib/src') },