dd-trace 2.11.0 → 3.0.0-pre.0

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -3,13 +3,25 @@ const { truncateSpan, normalizeSpan } = require('./tags-processors')
 const Chunk = require('./chunk')
 const { AgentEncoder } = require('./0.4')
 const { version: ddTraceVersion } = require('../../../../package.json')
+const id = require('../../../dd-trace/src/id')
 
 const ENCODING_VERSION = 1
 
+const ALLOWED_CONTENT_TYPES = ['test_session_end', 'test_suite_end', 'test']
+
+const TEST_SUITE_KEYS_LENGTH = 11
+const TEST_SESSION_KEYS_LENGTH = 10
+
+const CHUNK_SIZE = 4 * 1024 * 1024 // 4MB
+
 function formatSpan (span) {
+  let encodingVersion = ENCODING_VERSION
+  if (span.type === 'test' && span.meta && span.meta.test_session_id) {
+    encodingVersion = 2
+  }
   return {
-    type: span.type === 'test' ? 'test' : 'span',
-    version: ENCODING_VERSION,
+    type: ALLOWED_CONTENT_TYPES.includes(span.type) ? span.type : 'span',
+    version: encodingVersion,
     content: normalizeSpan(truncateSpan(span))
   }
 }
@@ -21,8 +33,8 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this.runtimeId = runtimeId
     this.service = service
     this.env = env
-    this._traceBytes = new Chunk()
-    this._stringBytes = new Chunk()
+    this._traceBytes = new Chunk(CHUNK_SIZE)
+    this._stringBytes = new Chunk(CHUNK_SIZE)
     this._stringCount = 0
     this._stringMap = {}
 
@@ -33,8 +45,69 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this.reset()
   }
 
+  _encodeTestSuite (bytes, content) {
+    this._encodeMapPrefix(bytes, TEST_SUITE_KEYS_LENGTH)
+    this._encodeString(bytes, 'type')
+    this._encodeString(bytes, content.type)
+
+    this._encodeString(bytes, 'test_session_id')
+    this._encodeId(bytes, content.trace_id)
+
+    this._encodeString(bytes, 'test_suite_id')
+    this._encodeId(bytes, content.span_id)
+
+    this._encodeString(bytes, 'error')
+    this._encodeNumber(bytes, content.error)
+    this._encodeString(bytes, 'name')
+    this._encodeString(bytes, content.name)
+    this._encodeString(bytes, 'service')
+    this._encodeString(bytes, content.service)
+    this._encodeString(bytes, 'resource')
+    this._encodeString(bytes, content.resource)
+    this._encodeString(bytes, 'start')
+    this._encodeNumber(bytes, content.start)
+    this._encodeString(bytes, 'duration')
+    this._encodeNumber(bytes, content.duration)
+    this._encodeString(bytes, 'meta')
+    this._encodeMap(bytes, content.meta)
+    this._encodeString(bytes, 'metrics')
+    this._encodeMap(bytes, content.metrics)
+  }
+
+  _encodeTestSession (bytes, content) {
+    this._encodeMapPrefix(bytes, TEST_SESSION_KEYS_LENGTH)
+    this._encodeString(bytes, 'type')
+    this._encodeString(bytes, content.type)
+
+    this._encodeString(bytes, 'test_session_id')
+    this._encodeId(bytes, content.trace_id)
+
+    this._encodeString(bytes, 'error')
+    this._encodeNumber(bytes, content.error)
+    this._encodeString(bytes, 'name')
+    this._encodeString(bytes, content.name)
+    this._encodeString(bytes, 'service')
+    this._encodeString(bytes, content.service)
+    this._encodeString(bytes, 'resource')
+    this._encodeString(bytes, content.resource)
+    this._encodeString(bytes, 'start')
+    this._encodeNumber(bytes, content.start)
+    this._encodeString(bytes, 'duration')
+    this._encodeNumber(bytes, content.duration)
+    this._encodeString(bytes, 'meta')
+    this._encodeMap(bytes, content.meta)
+    this._encodeString(bytes, 'metrics')
+    this._encodeMap(bytes, content.metrics)
+  }
+
   _encodeEventContent (bytes, content) {
-    this._encodeMapPrefix(bytes, content)
+    const keysLength = Object.keys(content).length
+    if (content.meta.test_session_id) {
+      this._encodeMapPrefix(bytes, keysLength + 2)
+    } else {
+      this._encodeMapPrefix(bytes, keysLength)
+    }
+
     if (content.type) {
       this._encodeString(bytes, 'type')
       this._encodeString(bytes, content.type)
@@ -57,6 +130,24 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this._encodeNumber(bytes, content.start)
     this._encodeString(bytes, 'duration')
     this._encodeNumber(bytes, content.duration)
+    /**
+     * We include `test_session_id` and `test_suite_id`
+     * in the root of the event by passing them via the `meta` dict.
+     * This is to avoid changing the span format in packages/dd-trace/src/format.js,
+     * which can have undesired side effects in other products.
+     * But `test_session_id` and `test_suite_id` are *not* supposed to be in `meta`,
+     * so we delete them before enconding the dictionary.
+     * TODO: find a better way to do this.
+     */
+    if (content.meta.test_session_id) {
+      this._encodeString(bytes, 'test_session_id')
+      this._encodeId(bytes, id(content.meta.test_session_id))
+      delete content.meta.test_session_id
+
+      this._encodeString(bytes, 'test_suite_id')
+      this._encodeId(bytes, id(content.meta.test_suite_id))
+      delete content.meta.test_suite_id
+    }
     this._encodeString(bytes, 'meta')
     this._encodeMap(bytes, content.meta)
     this._encodeString(bytes, 'metrics')
@@ -64,7 +155,7 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
   }
 
   _encodeEvent (bytes, event) {
-    this._encodeMapPrefix(bytes, event)
+    this._encodeMapPrefix(bytes, Object.keys(event).length)
     this._encodeString(bytes, 'type')
     this._encodeString(bytes, event.type)
 
@@ -72,7 +163,13 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this._encodeNumber(bytes, event.version)
 
     this._encodeString(bytes, 'content')
-    this._encodeEventContent(bytes, event.content)
+    if (event.type === 'span' || event.type === 'test') {
+      this._encodeEventContent(bytes, event.content)
+    } else if (event.type === 'test_suite_end') {
+      this._encodeTestSuite(bytes, event.content)
+    } else if (event.type === 'test_session_end') {
+      this._encodeTestSession(bytes, event.content)
+    }
   }
 
   _encodeNumber (bytes, value) {
@@ -107,18 +204,17 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     buffer[offset + 8] = lo
   }
 
-  _encodeMapPrefix (bytes, map) {
-    const keys = Object.keys(map)
+  _encodeMapPrefix (bytes, keysLength) {
     const buffer = bytes.buffer
     const offset = bytes.length
 
     bytes.reserve(5)
     bytes.length += 5
     buffer[offset] = 0xdf
-    buffer[offset + 1] = keys.length >> 24
-    buffer[offset + 2] = keys.length >> 16
-    buffer[offset + 3] = keys.length >> 8
-    buffer[offset + 4] = keys.length
+    buffer[offset + 1] = keysLength >> 24
+    buffer[offset + 2] = keysLength >> 16
+    buffer[offset + 3] = keysLength >> 8
+    buffer[offset + 4] = keysLength
   }
 
   _encode (bytes, trace) {
@@ -171,11 +267,11 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
       payload.metadata['*']['runtime-id'] = this.runtimeId
     }
 
-    this._encodeMapPrefix(bytes, payload)
+    this._encodeMapPrefix(bytes, Object.keys(payload).length)
     this._encodeString(bytes, 'version')
     this._encodeNumber(bytes, payload.version)
     this._encodeString(bytes, 'metadata')
-    this._encodeMapPrefix(bytes, payload.metadata)
+    this._encodeMapPrefix(bytes, Object.keys(payload.metadata).length)
     this._encodeString(bytes, '*')
     this._encodeMap(bytes, payload.metadata['*'])
     this._encodeString(bytes, 'events')

package/packages/dd-trace/src/exporters/common/request.js

@@ -1,22 +1,32 @@
 'use strict'
 
+// TODO: Add test with slow or unresponsive agent.
+// TODO: Add telemetry for things like dropped requests, errors, etc.
+
 const http = require('http')
 const https = require('https')
-const log = require('../../log')
 const docker = require('./docker')
 const { storage } = require('../../../../datadog-core')
 
-const httpAgent = new http.Agent({ keepAlive: true })
-const httpsAgent = new https.Agent({ keepAlive: true })
+const keepAlive = true
+const maxTotalSockets = 8
+const httpAgent = new http.Agent({ keepAlive, maxTotalSockets })
+const httpsAgent = new https.Agent({ keepAlive, maxTotalSockets })
 const containerId = docker.id()
 
+let activeRequests = 0
+
 function request (data, options, keepAlive, callback) {
   if (!options.headers) {
     options.headers = {}
   }
+
+  // The timeout should be kept low to avoid excessive queueing.
+  const timeout = options.timeout || 2000
   const isSecure = options.protocol === 'https:'
   const client = isSecure ? https : http
   const dataArray = [].concat(data)
+
   options.headers['Content-Length'] = byteLength(dataArray)
 
   if (containerId) {
@@ -27,39 +37,15 @@ function request (data, options, keepAlive, callback) {
     options.agent = isSecure ? httpsAgent : httpAgent
   }
 
-  const firstRequest = retriableRequest(options, client, callback)
-  dataArray.forEach(buffer => firstRequest.write(buffer))
-
-  // The first request will be retried
-  const firstRequestErrorHandler = () => {
-    log.debug('Retrying request to the intake')
-    const retriedReq = retriableRequest(options, client, callback)
-    dataArray.forEach(buffer => retriedReq.write(buffer))
-    // The retried request will fail normally
-    retriedReq.on('error', e => callback(new Error(`Network error trying to reach the intake: ${e.message}`)))
-    retriedReq.end()
-  }
-
-  firstRequest.on('error', firstRequestErrorHandler)
-  firstRequest.end()
-
-  return firstRequest
-}
-
-function retriableRequest (options, client, callback) {
-  const store = storage.getStore()
-
-  storage.enterWith({ noop: true })
-
-  const timeout = options.timeout || 15000
-
-  const request = client.request(options, res => {
+  const onResponse = res => {
     let responseData = ''
 
     res.setTimeout(timeout)
 
     res.on('data', chunk => { responseData += chunk })
     res.on('end', () => {
+      activeRequests--
+
       if (res.statusCode >= 200 && res.statusCode <= 299) {
         callback(null, responseData, res.statusCode)
       } else {
@@ -69,11 +55,34 @@ function retriableRequest (options, client, callback) {
         callback(error, null, res.statusCode)
       }
     })
-  })
-  request.setTimeout(timeout, request.abort)
-  storage.enterWith(store)
+  }
+
+  const makeRequest = onError => {
+    // Limit to 1 request by socket, otherwise drop payload.
+    if (activeRequests >= maxTotalSockets) return
+
+    activeRequests++
+
+    const store = storage.getStore()
+
+    storage.enterWith({ noop: true })
+
+    const req = client.request(options, onResponse)
+
+    req.once('error', err => {
+      activeRequests--
+      onError(err)
+    })
+
+    dataArray.forEach(buffer => req.write(buffer))
+
+    req.setTimeout(timeout, req.abort)
+    req.end()
+
+    storage.enterWith(store)
+  }
 
-  return request
+  makeRequest(() => makeRequest(callback))
 }
 
 function byteLength (data) {

package/packages/dd-trace/src/noop/tracer.js

@@ -24,10 +24,6 @@ class NoopTracer extends Tracer {
     return this._scope
   }
 
-  currentSpan () {
-    return null
-  }
-
   getRumData () {
     return ''
   }

package/packages/dd-trace/src/plugin_manager.js

@@ -36,61 +36,81 @@ const collectDisabledPlugins = () => {
 // TODO this must always be a singleton.
 module.exports = class PluginManager {
   constructor (tracer) {
+    this._tracer = tracer
     this._pluginsByName = {}
     this._configsByName = {}
-
-    const _disabledPlugins = collectDisabledPlugins()
-
-    for (const PluginClass of Object.values(plugins)) {
-      /**
-       * disabling the plugin here instead of in `configure` so we don't waste subscriber
-       * resources on a plugin that will eventually be disabled anyways
-       */
-      if (_disabledPlugins.has(PluginClass.name)) {
-        log.debug(`Plugin "${PluginClass.name}" was disabled via configuration option.`)
-        continue
-      }
-      if (typeof PluginClass !== 'function') continue
-      this._pluginsByName[PluginClass.name] = new PluginClass(tracer)
-      this._configsByName[PluginClass.name] = {}
-    }
+    this._disabledPlugins = collectDisabledPlugins()
   }
 
   // like instrumenter.use()
   configurePlugin (name, pluginConfig) {
-    if (!(name in this._pluginsByName)) return
     if (typeof pluginConfig === 'boolean') {
       pluginConfig = { enabled: pluginConfig }
     }
+    if (!pluginConfig) {
+      pluginConfig = { enabled: true }
+    }
 
-    const config = {
+    this._configsByName[name] = {
       ...this._configsByName[name],
       ...pluginConfig
     }
 
-    this._pluginsByName[name].configure(getConfig(name, config))
+    if (this._pluginsByName[name]) {
+      this._pluginsByName[name].configure(getConfig(name, this._configsByName[name]))
+    }
   }
 
   // like instrumenter.enable()
-  configure (config) {
-    const { logInjection, serviceMapping } = config
-
-    if (config.plugins !== false) {
-      for (const name in this._pluginsByName) {
-        const pluginConfig = {
-          ...this._configsByName[name],
-          logInjection
-        }
-        if (serviceMapping && serviceMapping[name]) {
-          pluginConfig.service = serviceMapping[name]
-        }
-        this.configurePlugin(name, pluginConfig)
+  configure (config = {}) {
+    const { logInjection, serviceMapping, experimental, queryStringObfuscation } = config
+
+    for (const PluginClass of Object.values(plugins)) {
+      const name = PluginClass.name
+
+      if (this._disabledPlugins.has(name)) {
+        log.debug(`Plugin "${name}" was disabled via configuration option.`)
+        continue
+      }
+
+      if (typeof PluginClass !== 'function') continue
+
+      this._pluginsByName[name] = new PluginClass(this._tracer)
+
+      if (config.plugins === false) continue
+
+      const pluginConfig = {
+        ...this._configsByName[name]
+      }
+
+      if (logInjection !== undefined) {
+        pluginConfig.logInjection = logInjection
+      }
+
+      if (queryStringObfuscation !== undefined) {
+        pluginConfig.queryStringObfuscation = queryStringObfuscation
+      }
+
+      // TODO: update so that it's available for every CI Visibility's plugin
+      if (name === 'mocha') {
+        pluginConfig.isAgentlessEnabled = experimental && experimental.exporter === 'datadog'
+      }
+
+      if (serviceMapping && serviceMapping[name]) {
+        pluginConfig.service = serviceMapping[name]
       }
+
+      this.configurePlugin(name, pluginConfig)
     }
   }
 
   // This is basically just for testing. like intrumenter.disable()
   destroy () {
-    for (const name in this._pluginsByName) this._pluginsByName[name].configure({ enabled: false })
+    for (const name in this._pluginsByName) {
+      this._pluginsByName[name].configure({ enabled: false })
+    }
+
+    this._pluginsByName = {}
+    this._configsByName = {}
   }
 }

package/packages/dd-trace/src/plugins/index.js

@@ -15,7 +15,6 @@ module.exports = {
   'express': require('../../../datadog-plugin-express/src'),
   'fastify': require('../../../datadog-plugin-fastify/src'),
   'find-my-way': require('../../../datadog-plugin-find-my-way/src'),
-  'fs': require('../../../datadog-plugin-fs/src'),
   'google-cloud-pubsub': require('../../../datadog-plugin-google-cloud-pubsub/src'),
   'graphql': require('../../../datadog-plugin-graphql/src'),
   'grpc': require('../../../datadog-plugin-grpc/src'),

package/packages/dd-trace/src/plugins/util/test.js

@@ -36,6 +36,9 @@ const TEST_IS_RUM_ACTIVE = 'test.is_rum_active'
 const TEST_CODE_OWNERS = 'test.codeowners'
 const TEST_SOURCE_FILE = 'test.source.file'
 const LIBRARY_VERSION = 'library_version'
+const TEST_COMMAND = 'test.command'
+const TEST_SESSION_ID = 'test_session_id'
+const TEST_SUITE_ID = 'test_suite_id'
 
 const ERROR_TYPE = 'error.type'
 const ERROR_MESSAGE = 'error.msg'
@@ -70,7 +73,12 @@ module.exports = {
   getTestSuitePath,
   getCodeOwnersFileEntries,
   getCodeOwnersForFilename,
-  getTestCommonTags
+  getTestCommonTags,
+  getTestSessionCommonTags,
+  getTestSuiteCommonTags,
+  TEST_COMMAND,
+  TEST_SESSION_ID,
+  TEST_SUITE_ID
 }
 
 function getTestEnvironmentMetadata (testFramework, config) {
@@ -225,3 +233,26 @@ function getCodeOwnersForFilename (filename, entries) {
   }
   return null
 }
+
+function getTestSessionCommonTags (command, version) {
+  return {
+    [SPAN_TYPE]: 'test_session_end',
+    [TEST_TYPE]: 'test',
+    [RESOURCE_NAME]: `test_session.${command}`,
+    [TEST_FRAMEWORK_VERSION]: version,
+    [LIBRARY_VERSION]: ddTraceVersion,
+    [TEST_COMMAND]: command
+  }
+}
+
+function getTestSuiteCommonTags (command, version, testSuite) {
+  return {
+    [SPAN_TYPE]: 'test_suite_end',
+    [TEST_TYPE]: 'test',
+    [RESOURCE_NAME]: `test_suite.${testSuite}`,
+    [TEST_FRAMEWORK_VERSION]: version,
+    [LIBRARY_VERSION]: ddTraceVersion,
+    [TEST_SUITE]: testSuite,
+    [TEST_COMMAND]: command
+  }
+}

package/packages/dd-trace/src/plugins/util/web.js

@@ -43,13 +43,15 @@ const web = {
     const hooks = getHooks(config)
     const filter = urlFilter.getFilter(config)
     const middleware = getMiddlewareSetting(config)
+    const queryStringObfuscation = getQsObfuscator(config)
 
     return Object.assign({}, config, {
       headers,
       validateStatus,
       hooks,
       filter,
-      middleware
+      middleware,
+      queryStringObfuscation
     })
   },
 
@@ -259,9 +261,9 @@ const web = {
     const context = contexts.get(req)
     const span = context.span
     const error = context.error
-    const hasMiddlewareError = span.context()._tags['error'] || span.context()._tags['error.msg']
+    const hasExistingError = span.context()._tags['error'] || span.context()._tags['error.msg']
 
-    if (!hasMiddlewareError && !context.config.validateStatus(statusCode)) {
+    if (!hasExistingError && !context.config.validateStatus(statusCode)) {
       span.setTag(ERROR, error || true)
     }
   },
@@ -270,7 +272,10 @@ const web = {
   addError (req, error) {
     if (error instanceof Error) {
       const context = contexts.get(req)
-      context.error = context.error || error
+
+      if (context) {
+        context.error = error
+      }
     }
   },
 
@@ -298,6 +303,40 @@ const web = {
     context.span.finish()
     context.finished = true
   },
+
+  finishAll (context) {
+    const { req, res } = context
+
+    for (const beforeEnd of context.beforeEnd) {
+      beforeEnd()
+    }
+
+    web.finishMiddleware(context)
+
+    if (incomingHttpRequestEnd.hasSubscribers) {
+      incomingHttpRequestEnd.publish({ req, res })
+    }
+
+    web.finishSpan(context)
+  },
+
+  obfuscateQs (config, url) {
+    const { queryStringObfuscation } = config
+
+    if (queryStringObfuscation === false) return url
+
+    const i = url.indexOf('?')
+    if (i === -1) return url
+
+    const path = url.slice(0, i)
+    if (queryStringObfuscation === true) return path
+
+    let qs = url.slice(i + 1)
+
+    qs = qs.replace(queryStringObfuscation, '<redacted>')
+
+    return `${path}?${qs}`
+  },
   wrapWriteHead (context) {
     const { req, res } = context
     const writeHead = res.writeHead
@@ -318,21 +357,9 @@ const web = {
   },
   wrapRes (context, req, res, end) {
     return function () {
-      for (const beforeEnd of context.beforeEnd) {
-        beforeEnd()
-      }
-
-      web.finishMiddleware(context)
+      web.finishAll(context)
 
-      if (incomingHttpRequestEnd.hasSubscribers) {
-        incomingHttpRequestEnd.publish({ req, res })
-      }
-
-      const returnValue = end.apply(res, arguments)
-
-      web.finishSpan(context)
-
-      return returnValue
+      return end.apply(res, arguments)
     }
   },
   wrapEnd (context) {
@@ -399,11 +426,11 @@ function reactivate (req, fn) {
 }
 
 function addRequestTags (context) {
-  const { req, span } = context
+  const { req, span, config } = context
   const url = extractURL(req)
 
   span.addTags({
-    [HTTP_URL]: url.split('?')[0],
+    [HTTP_URL]: web.obfuscateQs(config, url),
     [HTTP_METHOD]: req.method,
     [SPAN_KIND]: SERVER,
     [SPAN_TYPE]: WEB,
@@ -514,4 +541,30 @@ function getMiddlewareSetting (config) {
   return true
 }
 
+function getQsObfuscator (config) {
+  const obfuscator = config.queryStringObfuscation
+
+  if (typeof obfuscator === 'boolean') {
+    return obfuscator
+  }
+
+  if (typeof obfuscator === 'string') {
+    if (obfuscator === '') return false // disable obfuscator
+
+    if (obfuscator === '.*') return true // optimize full redact
+
+    try {
+      return new RegExp(obfuscator, 'gi')
+    } catch (err) {
+      log.error(err)
+    }
+  }
+
+  if (config.hasOwnProperty('queryStringObfuscation')) {
+    log.error('Expected `queryStringObfuscation` to be a regex string or boolean.')
+  }
+
+  return true
+}
+
 module.exports = web