dd-trace 2.12.0 → 3.0.0-pre.1

package/ci/init.js

@@ -1,10 +1,7 @@
 /* eslint-disable no-console */
-
-const path = require('path')
 const tracer = require('../packages/dd-trace')
 const { ORIGIN_KEY } = require('../packages/dd-trace/src/constants')
-const { mochaHook } = require('../packages/datadog-instrumentations/src/mocha')
-const { pickleHook, testCaseHook } = require('../packages/datadog-instrumentations/src/cucumber')
+const { isTrue } = require('../packages/dd-trace/src/util')
 
 const options = {
   startupLogs: false,
@@ -15,9 +12,7 @@ const options = {
 
 let shouldInit = true
 
-const isAgentlessEnabled = process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED &&
-  process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED !== 'false' &&
-  process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED !== '0'
+const isAgentlessEnabled = isTrue(process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED)
 
 if (isAgentlessEnabled) {
   if (process.env.DATADOG_API_KEY || process.env.DD_API_KEY) {
@@ -32,25 +27,6 @@ so dd-trace will not be initialized.`)
   }
 }
 
-// TODO: remove this in a later major version since we now recommend using
-// `NODE_OPTIONS='-r dd-trace/ci/init'`.
-try {
-  for (const filename in require.cache) {
-    const cache = require.cache[filename]
-    const id = filename.split(path.sep).join('/')
-
-    if (id.includes('/node_modules/mocha/lib/runner.js')) {
-      cache.exports = mochaHook(cache.exports)
-    } else if (id.includes('/node_modules/@cucumber/cucumber/lib/runtime/pickle_runner.js')) {
-      cache.exports = pickleHook(cache.exports)
-    } else if (id.includes('/node_modules/@cucumber/cucumber/lib/runtime/test_case_runner.js')) {
-      cache.exports = testCaseHook(cache.exports)
-    }
-  }
-} catch (e) {
-  // ignore error and let the tracer initialize anyway
-}
-
 if (shouldInit) {
   tracer.init(options)
   tracer.use('fs', false)

package/ci/jest/env.js

@@ -2,6 +2,7 @@
 
 const tracer = require('../../packages/dd-trace')
 const { ORIGIN_KEY } = require('../../packages/dd-trace/src/constants')
+const { isTrue } = require('../../packages/dd-trace/src/util')
 
 const options = {
   startupLogs: false,
@@ -10,9 +11,7 @@ const options = {
   }
 }
 
-const isAgentlessEnabled = process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED &&
-  process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED !== 'false' &&
-  process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED !== '0'
+const isAgentlessEnabled = isTrue(process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED)
 
 if (isAgentlessEnabled) {
   if (process.env.DATADOG_API_KEY || process.env.DD_API_KEY) {

package/index.d.ts

@@ -562,7 +562,6 @@ interface Plugins {
   "elasticsearch": plugins.elasticsearch;
   "express": plugins.express;
   "fastify": plugins.fastify;
-  "fs": plugins.fs;
   "generic-pool": plugins.generic_pool;
   "google-cloud-pubsub": plugins.google_cloud_pubsub;
   "graphql": plugins.graphql;
@@ -903,12 +902,6 @@ declare namespace plugins {
    */
   interface fastify extends HttpServer {}
 
-  /**
-   * This plugin automatically instruments the
-   * [fs](https://nodejs.org/api/fs.html) module.
-   */
-  interface fs extends Instrumentation {}
-
   /**
    * This plugin patches the [generic-pool](https://github.com/coopernurse/node-pool)
    * module to bind the callbacks the the caller context.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "2.12.0",
+  "version": "3.0.0-pre.1",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",

package/packages/datadog-instrumentations/src/connect.js

@@ -78,12 +78,12 @@ function wrapLayerHandle (layer) {
 
       try {
         return original.apply(this, arguments)
-      } catch (e) {
-        errorChannel.publish(e)
+      } catch (error) {
+        errorChannel.publish({ req, error })
         nextChannel.publish({ req })
         exitChannel.publish({ req })
 
-        throw e
+        throw error
       }
     })
   })
@@ -92,7 +92,7 @@ function wrapLayerHandle (layer) {
 function wrapNext (req, next) {
   return function (error) {
     if (error) {
-      errorChannel.publish(error)
+      errorChannel.publish({ req, error })
     }
 
     nextChannel.publish({ req })

package/packages/datadog-instrumentations/src/fastify.js

@@ -6,7 +6,6 @@ const { addHook, channel, AsyncResource } = require('./helpers/instrument')
 const errorChannel = channel('apm:fastify:middleware:error')
 const handleChannel = channel('apm:fastify:request:handle')
 
-const requestResources = new WeakMap()
 const parsingResources = new WeakMap()
 
 function wrapFastify (fastify, hasParsingEvents) {
@@ -42,16 +41,13 @@ function wrapAddHook (addHook) {
 
     arguments[arguments.length - 1] = shimmer.wrap(fn, function (request, reply, done) {
       const req = getReq(request)
-      const requestResource = requestResources.get(req)
-
-      if (!requestResource) return fn.apply(this, arguments)
 
       try {
         if (typeof done === 'function') {
           done = arguments[arguments.length - 1]
 
           arguments[arguments.length - 1] = function (err) {
-            publishError(err, requestResource)
+            publishError(err, req)
 
             if (name === 'onRequest' || name === 'preParsing') {
               const parsingResource = new AsyncResource('bound-anonymous-fn')
@@ -71,13 +67,13 @@ function wrapAddHook (addHook) {
           const promise = fn.apply(this, arguments)
 
           if (promise && typeof promise.catch === 'function') {
-            return promise.catch(err => publishError(err, requestResource))
+            return promise.catch(err => publishError(err, req))
           }
 
           return promise
         }
       } catch (e) {
-        throw publishError(e, requestResource)
+        throw publishError(e, req)
       }
     })
 
@@ -90,14 +86,10 @@ function onRequest (request, reply, done) {
 
   const req = getReq(request)
   const res = getRes(reply)
-  const requestResource = new AsyncResource('bound-anonymous-fn')
 
-  requestResources.set(req, requestResource)
+  handleChannel.publish({ req, res })
 
-  return requestResource.runInAsyncScope(() => {
-    handleChannel.publish({ req, res })
-    return done()
-  })
+  return done()
 }
 
 function preHandler (request, reply, done) {
@@ -105,9 +97,8 @@ function preHandler (request, reply, done) {
   if (!reply || typeof reply.send !== 'function') return done()
 
   const req = getReq(request)
-  const requestResource = requestResources.get(req)
 
-  reply.send = wrapSend(reply.send, requestResource)
+  reply.send = wrapSend(reply.send, req)
 
   done()
 }
@@ -133,12 +124,10 @@ function preParsing (request, reply, payload, done) {
   parsingResource.runInAsyncScope(() => done())
 }
 
-function wrapSend (send, resource) {
-  return function sendWithTrace (payload) {
-    if (payload instanceof Error) {
-      resource.runInAsyncScope(() => {
-        errorChannel.publish(payload)
-      })
+function wrapSend (send, req) {
+  return function sendWithTrace (error) {
+    if (error instanceof Error) {
+      errorChannel.publish({ req, error })
     }
 
     return send.apply(this, arguments)
@@ -153,11 +142,9 @@ function getRes (reply) {
   return reply && (reply.raw || reply.res || reply)
 }
 
-function publishError (error, resource) {
+function publishError (error, req) {
   if (error) {
-    resource.runInAsyncScope(() => {
-      errorChannel.publish(error)
-    })
+    errorChannel.publish({ error, req })
   }
 
   return error

package/packages/datadog-instrumentations/src/koa.js

@@ -126,13 +126,13 @@ function wrapMiddleware (fn, layer) {
 }
 
 function fulfill (ctx, error) {
-  if (error) {
-    errorChannel.publish(error)
-  }
-
   const req = ctx.req
   const route = ctx.routePath
 
+  if (error) {
+    errorChannel.publish({ req, error })
+  }
+
   // TODO: make sure that the parent class cannot override this in `enter`
   if (route) {
     routeChannel.publish({ req, route })

package/packages/datadog-instrumentations/src/router.js

@@ -49,12 +49,12 @@ function createWrapRouterMethod (name) {
 
         try {
           return original.apply(this, arguments)
-        } catch (e) {
-          errorChannel.publish(e)
+        } catch (error) {
+          errorChannel.publish({ req, error })
           nextChannel.publish({ req })
           exitChannel.publish({ req })
 
-          throw e
+          throw error
         }
       })
     })
@@ -91,7 +91,7 @@ function createWrapRouterMethod (name) {
   function wrapNext (req, next) {
     return function (error) {
       if (error) {
-        errorChannel.publish(error)
+        errorChannel.publish({ req, error })
       }
 
       nextChannel.publish({ req })

package/packages/datadog-plugin-grpc/src/client.js

@@ -19,13 +19,13 @@ class GrpcClientPlugin extends Plugin {
       const metadataFilter = this.config.metadataFilter
       const store = storage.getStore()
       const childOf = store && store.span
-      const span = this.tracer.startSpan('grpc.request', {
+      const span = this.tracer.startSpan('grpc.client', {
         childOf,
         tags: {
           [Tags.SPAN_KIND]: 'client',
           'span.type': 'http',
           'resource.name': path,
-          'service.name': this.config.service || `${this.tracer._service}-grpc-client`,
+          'service.name': this.config.service || this.tracer._service,
           'component': 'grpc'
         }
       })

package/packages/datadog-plugin-grpc/src/server.js

@@ -19,13 +19,13 @@ class GrpcServerPlugin extends Plugin {
       const metadataFilter = this.config.metadataFilter
       const store = storage.getStore()
       const childOf = extract(this.tracer, metadata)
-      const span = this.tracer.startSpan('grpc.request', {
+      const span = this.tracer.startSpan('grpc.server', {
         childOf,
         tags: {
           [Tags.SPAN_KIND]: 'server',
           'span.type': 'web',
           'resource.name': name,
-          'service.name': this.config.service || `${this.tracer._service}`,
+          'service.name': this.config.service || this.tracer._service,
           'component': 'grpc'
         }
       })

package/packages/datadog-plugin-http/src/client.js

@@ -190,7 +190,7 @@ function getServiceName (tracer, config, options) {
     return config.service
   }
 
-  return `${tracer._service}-http-client`
+  return tracer._service
 }
 
 function getHost (options) {

package/packages/datadog-plugin-http/src/server.js

@@ -15,7 +15,7 @@ class HttpServerPlugin extends Plugin {
 
     this.addSub('apm:http:server:request:start', ({ req, res }) => {
       const store = storage.getStore()
-      const span = web.startSpan(this.tracer, this.config, req, res, 'http.request')
+      const span = web.startSpan(this.tracer, this.config, req, res, 'web.request')
 
       this.enter(span, { ...store, req })
 

package/packages/datadog-plugin-http2/src/client.js

@@ -118,7 +118,7 @@ function getServiceName (tracer, config, sessionDetails) {
     return config.service
   }
 
-  return `${tracer._service}-http-client`
+  return tracer._service
 }
 
 function hasAmazonSignature (headers, path) {

package/packages/datadog-plugin-router/src/index.js

@@ -16,7 +16,10 @@ class RouterPlugin extends WebPlugin {
     this._contexts = new WeakMap()
 
     this.addSub(`apm:${this.constructor.name}:middleware:enter`, ({ req, name, route }) => {
-      const childOf = this._getActive(req)
+      const childOf = this._getActive(req) || this._getStoreSpan()
+
+      if (!childOf) return
+
       const span = this._getMiddlewareSpan(name, childOf)
       const context = this._createContext(req, route, childOf)
 
@@ -46,14 +49,16 @@ class RouterPlugin extends WebPlugin {
       context.middleware.pop().finish()
     })
 
-    this.addSub(`apm:${this.constructor.name}:middleware:error`, err => {
-      const store = storage.getStore()
+    this.addSub(`apm:${this.constructor.name}:middleware:error`, ({ req, error }) => {
+      web.addError(req, error)
 
-      web.addError(store.req, err)
+      if (!this.config.middleware) return
 
-      if (this.config.middleware) {
-        this.addError(err)
-      }
+      const span = this._getActive(req)
+
+      if (!span) return
+
+      span.setTag('error', error)
     })
 
     this.addSub(`apm:http:server:request:finish`, ({ req }) => {
@@ -72,7 +77,7 @@ class RouterPlugin extends WebPlugin {
   _getActive (req) {
     const context = this._contexts.get(req)
 
-    if (!context) return this._getStoreSpan()
+    if (!context) return
     if (context.middleware.length === 0) return context.span
 
     return context.middleware[context.middleware.length - 1]

package/packages/dd-trace/src/config.js

@@ -13,6 +13,9 @@ const uuid = require('crypto-randomuuid')
 const fromEntries = Object.fromEntries || (entries =>
   entries.reduce((obj, [k, v]) => Object.assign(obj, { [k]: v }), {}))
 
+// eslint-disable-next-line max-len
+const qsRegex = '(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:(?:\\s|%20)*(?:=|%3D)[^&]+|(?:"|%22)(?:\\s|%20)*(?::|%3A)(?:\\s|%20)*(?:"|%22)(?:%2[^2]|%[^2]|[^"%])+(?:"|%22))|bearer(?:\\s|%20)+[a-z0-9\\._\\-]|token(?::|%3A)[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L](?:[\\w=-]|%3D)+\\.ey[I-L](?:[\\w=-]|%3D)+(?:\\.(?:[\\w.+\\/=-]|%3D|%2F|%2B)+)?|[\\-]{5}BEGIN(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY[\\-]{5}[^\\-]+[\\-]{5}END(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY|ssh-rsa(?:\\s|%20)*(?:[a-z0-9\\/\\.+]|%2F|%5C|%2B){100,}'
+
 class Config {
   constructor (options) {
     options = options || {}
@@ -109,6 +112,10 @@ class Config {
       parseInt(process.env.DD_TRACE_PARTIAL_FLUSH_MIN_SPANS),
       1000
     )
+    const DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP = coalesce(
+      process.env.DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP,
+      qsRegex
+    )
     const DD_TRACE_B3_ENABLED = coalesce(
       options.experimental && options.experimental.b3,
       process.env.DD_TRACE_EXPERIMENTAL_B3_ENABLED,
@@ -203,6 +210,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     this.flushInterval = coalesce(parseInt(options.flushInterval, 10), defaultFlushInterval)
     this.flushMinSpans = DD_TRACE_PARTIAL_FLUSH_MIN_SPANS
     this.sampleRate = coalesce(Math.min(Math.max(sampler.sampleRate, 0), 1), 1)
+    this.queryStringObfuscation = DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP
     this.logger = options.logger
     this.plugins = !!coalesce(options.plugins, true)
     this.service = DD_SERVICE

package/packages/dd-trace/src/encode/0.4.js

@@ -57,6 +57,10 @@ class AgentEncoder {
     return buffer
   }
 
+  reset () {
+    this._reset()
+  }
+
   _encode (bytes, trace) {
     this._encodeArrayPrefix(bytes, trace)
 

package/packages/dd-trace/src/exporters/common/request.js

@@ -1,22 +1,32 @@
 'use strict'
 
+// TODO: Add test with slow or unresponsive agent.
+// TODO: Add telemetry for things like dropped requests, errors, etc.
+
 const http = require('http')
 const https = require('https')
-const log = require('../../log')
 const docker = require('./docker')
 const { storage } = require('../../../../datadog-core')
 
-const httpAgent = new http.Agent({ keepAlive: true })
-const httpsAgent = new https.Agent({ keepAlive: true })
+const keepAlive = true
+const maxTotalSockets = 8
+const httpAgent = new http.Agent({ keepAlive, maxTotalSockets })
+const httpsAgent = new https.Agent({ keepAlive, maxTotalSockets })
 const containerId = docker.id()
 
+let activeRequests = 0
+
 function request (data, options, keepAlive, callback) {
   if (!options.headers) {
     options.headers = {}
   }
+
+  // The timeout should be kept low to avoid excessive queueing.
+  const timeout = options.timeout || 2000
   const isSecure = options.protocol === 'https:'
   const client = isSecure ? https : http
   const dataArray = [].concat(data)
+
   options.headers['Content-Length'] = byteLength(dataArray)
 
   if (containerId) {
@@ -27,39 +37,15 @@ function request (data, options, keepAlive, callback) {
     options.agent = isSecure ? httpsAgent : httpAgent
   }
 
-  const firstRequest = retriableRequest(options, client, callback)
-  dataArray.forEach(buffer => firstRequest.write(buffer))
-
-  // The first request will be retried
-  const firstRequestErrorHandler = () => {
-    log.debug('Retrying request to the intake')
-    const retriedReq = retriableRequest(options, client, callback)
-    dataArray.forEach(buffer => retriedReq.write(buffer))
-    // The retried request will fail normally
-    retriedReq.on('error', e => callback(new Error(`Network error trying to reach the intake: ${e.message}`)))
-    retriedReq.end()
-  }
-
-  firstRequest.on('error', firstRequestErrorHandler)
-  firstRequest.end()
-
-  return firstRequest
-}
-
-function retriableRequest (options, client, callback) {
-  const store = storage.getStore()
-
-  storage.enterWith({ noop: true })
-
-  const timeout = options.timeout || 15000
-
-  const request = client.request(options, res => {
+  const onResponse = res => {
     let responseData = ''
 
     res.setTimeout(timeout)
 
     res.on('data', chunk => { responseData += chunk })
     res.on('end', () => {
+      activeRequests--
+
       if (res.statusCode >= 200 && res.statusCode <= 299) {
         callback(null, responseData, res.statusCode)
       } else {
@@ -69,15 +55,43 @@ function retriableRequest (options, client, callback) {
         callback(error, null, res.statusCode)
       }
     })
-  })
-  request.setTimeout(timeout, request.abort)
-  storage.enterWith(store)
+  }
 
-  return request
+  const makeRequest = onError => {
+    if (!request.writable) return callback(null)
+
+    activeRequests++
+
+    const store = storage.getStore()
+
+    storage.enterWith({ noop: true })
+
+    const req = client.request(options, onResponse)
+
+    req.once('error', err => {
+      activeRequests--
+      onError(err)
+    })
+
+    dataArray.forEach(buffer => req.write(buffer))
+
+    req.setTimeout(timeout, req.abort)
+    req.end()
+
+    storage.enterWith(store)
+  }
+
+  makeRequest(() => makeRequest(callback))
 }
 
 function byteLength (data) {
   return data.length > 0 ? data.reduce((prev, next) => prev + next.length, 0) : 0
 }
 
+Object.defineProperty(request, 'writable', {
+  get () {
+    return activeRequests < maxTotalSockets
+  }
+})
+
 module.exports = request

package/packages/dd-trace/src/exporters/common/writer.js

@@ -1,4 +1,6 @@
 'use strict'
+
+const request = require('./request')
 const log = require('../../log')
 
 class Writer {
@@ -9,7 +11,10 @@ class Writer {
   flush (done = () => {}) {
     const count = this._encoder.count()
 
-    if (count > 0) {
+    if (!request.writable) {
+      this._encoder.reset()
+      done()
+    } else if (count > 0) {
       const payload = this._encoder.makePayload()
 
       this._sendPayload(payload, count, done)
@@ -19,6 +24,8 @@ class Writer {
   }
 
   append (spans) {
+    if (!request.writable) return
+
     log.debug(() => `Encoding trace: ${JSON.stringify(spans)}`)
 
     this._encode(spans)

package/packages/dd-trace/src/noop/tracer.js

@@ -24,10 +24,6 @@ class NoopTracer extends Tracer {
     return this._scope
   }
 
-  currentSpan () {
-    return null
-  }
-
   getRumData () {
     return ''
   }

package/packages/dd-trace/src/plugin_manager.js

@@ -63,7 +63,7 @@ module.exports = class PluginManager {
 
   // like instrumenter.enable()
   configure (config = {}) {
-    const { logInjection, serviceMapping, experimental } = config
+    const { logInjection, serviceMapping, experimental, queryStringObfuscation } = config
 
     for (const PluginClass of Object.values(plugins)) {
       const name = PluginClass.name
@@ -87,6 +87,10 @@ module.exports = class PluginManager {
         pluginConfig.logInjection = logInjection
       }
 
+      if (queryStringObfuscation !== undefined) {
+        pluginConfig.queryStringObfuscation = queryStringObfuscation
+      }
+
       // TODO: update so that it's available for every CI Visibility's plugin
       if (name === 'mocha') {
         pluginConfig.isAgentlessEnabled = experimental && experimental.exporter === 'datadog'

package/packages/dd-trace/src/plugins/index.js

@@ -15,7 +15,6 @@ module.exports = {
   'express': require('../../../datadog-plugin-express/src'),
   'fastify': require('../../../datadog-plugin-fastify/src'),
   'find-my-way': require('../../../datadog-plugin-find-my-way/src'),
-  'fs': require('../../../datadog-plugin-fs/src'),
   'google-cloud-pubsub': require('../../../datadog-plugin-google-cloud-pubsub/src'),
   'graphql': require('../../../datadog-plugin-graphql/src'),
   'grpc': require('../../../datadog-plugin-grpc/src'),