dd-trace 2.11.0 → 2.12.1

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -3,13 +3,25 @@ const { truncateSpan, normalizeSpan } = require('./tags-processors')
 const Chunk = require('./chunk')
 const { AgentEncoder } = require('./0.4')
 const { version: ddTraceVersion } = require('../../../../package.json')
+const id = require('../../../dd-trace/src/id')
 
 const ENCODING_VERSION = 1
 
+const ALLOWED_CONTENT_TYPES = ['test_session_end', 'test_suite_end', 'test']
+
+const TEST_SUITE_KEYS_LENGTH = 11
+const TEST_SESSION_KEYS_LENGTH = 10
+
+const CHUNK_SIZE = 4 * 1024 * 1024 // 4MB
+
 function formatSpan (span) {
+  let encodingVersion = ENCODING_VERSION
+  if (span.type === 'test' && span.meta && span.meta.test_session_id) {
+    encodingVersion = 2
+  }
   return {
-    type: span.type === 'test' ? 'test' : 'span',
-    version: ENCODING_VERSION,
+    type: ALLOWED_CONTENT_TYPES.includes(span.type) ? span.type : 'span',
+    version: encodingVersion,
     content: normalizeSpan(truncateSpan(span))
   }
 }
@@ -21,8 +33,8 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this.runtimeId = runtimeId
     this.service = service
     this.env = env
-    this._traceBytes = new Chunk()
-    this._stringBytes = new Chunk()
+    this._traceBytes = new Chunk(CHUNK_SIZE)
+    this._stringBytes = new Chunk(CHUNK_SIZE)
     this._stringCount = 0
     this._stringMap = {}
 
@@ -33,8 +45,69 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this.reset()
   }
 
+  _encodeTestSuite (bytes, content) {
+    this._encodeMapPrefix(bytes, TEST_SUITE_KEYS_LENGTH)
+    this._encodeString(bytes, 'type')
+    this._encodeString(bytes, content.type)
+
+    this._encodeString(bytes, 'test_session_id')
+    this._encodeId(bytes, content.trace_id)
+
+    this._encodeString(bytes, 'test_suite_id')
+    this._encodeId(bytes, content.span_id)
+
+    this._encodeString(bytes, 'error')
+    this._encodeNumber(bytes, content.error)
+    this._encodeString(bytes, 'name')
+    this._encodeString(bytes, content.name)
+    this._encodeString(bytes, 'service')
+    this._encodeString(bytes, content.service)
+    this._encodeString(bytes, 'resource')
+    this._encodeString(bytes, content.resource)
+    this._encodeString(bytes, 'start')
+    this._encodeNumber(bytes, content.start)
+    this._encodeString(bytes, 'duration')
+    this._encodeNumber(bytes, content.duration)
+    this._encodeString(bytes, 'meta')
+    this._encodeMap(bytes, content.meta)
+    this._encodeString(bytes, 'metrics')
+    this._encodeMap(bytes, content.metrics)
+  }
+
+  _encodeTestSession (bytes, content) {
+    this._encodeMapPrefix(bytes, TEST_SESSION_KEYS_LENGTH)
+    this._encodeString(bytes, 'type')
+    this._encodeString(bytes, content.type)
+
+    this._encodeString(bytes, 'test_session_id')
+    this._encodeId(bytes, content.trace_id)
+
+    this._encodeString(bytes, 'error')
+    this._encodeNumber(bytes, content.error)
+    this._encodeString(bytes, 'name')
+    this._encodeString(bytes, content.name)
+    this._encodeString(bytes, 'service')
+    this._encodeString(bytes, content.service)
+    this._encodeString(bytes, 'resource')
+    this._encodeString(bytes, content.resource)
+    this._encodeString(bytes, 'start')
+    this._encodeNumber(bytes, content.start)
+    this._encodeString(bytes, 'duration')
+    this._encodeNumber(bytes, content.duration)
+    this._encodeString(bytes, 'meta')
+    this._encodeMap(bytes, content.meta)
+    this._encodeString(bytes, 'metrics')
+    this._encodeMap(bytes, content.metrics)
+  }
+
   _encodeEventContent (bytes, content) {
-    this._encodeMapPrefix(bytes, content)
+    const keysLength = Object.keys(content).length
+    if (content.meta.test_session_id) {
+      this._encodeMapPrefix(bytes, keysLength + 2)
+    } else {
+      this._encodeMapPrefix(bytes, keysLength)
+    }
+
     if (content.type) {
       this._encodeString(bytes, 'type')
       this._encodeString(bytes, content.type)
@@ -57,6 +130,24 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this._encodeNumber(bytes, content.start)
     this._encodeString(bytes, 'duration')
     this._encodeNumber(bytes, content.duration)
+    /**
+     * We include `test_session_id` and `test_suite_id`
+     * in the root of the event by passing them via the `meta` dict.
+     * This is to avoid changing the span format in packages/dd-trace/src/format.js,
+     * which can have undesired side effects in other products.
+     * But `test_session_id` and `test_suite_id` are *not* supposed to be in `meta`,
+     * so we delete them before enconding the dictionary.
+     * TODO: find a better way to do this.
+     */
+    if (content.meta.test_session_id) {
+      this._encodeString(bytes, 'test_session_id')
+      this._encodeId(bytes, id(content.meta.test_session_id))
+      delete content.meta.test_session_id
+
+      this._encodeString(bytes, 'test_suite_id')
+      this._encodeId(bytes, id(content.meta.test_suite_id))
+      delete content.meta.test_suite_id
+    }
     this._encodeString(bytes, 'meta')
     this._encodeMap(bytes, content.meta)
     this._encodeString(bytes, 'metrics')
@@ -64,7 +155,7 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
   }
 
   _encodeEvent (bytes, event) {
-    this._encodeMapPrefix(bytes, event)
+    this._encodeMapPrefix(bytes, Object.keys(event).length)
     this._encodeString(bytes, 'type')
     this._encodeString(bytes, event.type)
 
@@ -72,7 +163,13 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this._encodeNumber(bytes, event.version)
 
     this._encodeString(bytes, 'content')
-    this._encodeEventContent(bytes, event.content)
+    if (event.type === 'span' || event.type === 'test') {
+      this._encodeEventContent(bytes, event.content)
+    } else if (event.type === 'test_suite_end') {
+      this._encodeTestSuite(bytes, event.content)
+    } else if (event.type === 'test_session_end') {
+      this._encodeTestSession(bytes, event.content)
+    }
   }
 
   _encodeNumber (bytes, value) {
@@ -107,18 +204,17 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     buffer[offset + 8] = lo
   }
 
-  _encodeMapPrefix (bytes, map) {
-    const keys = Object.keys(map)
+  _encodeMapPrefix (bytes, keysLength) {
     const buffer = bytes.buffer
     const offset = bytes.length
 
     bytes.reserve(5)
     bytes.length += 5
     buffer[offset] = 0xdf
-    buffer[offset + 1] = keys.length >> 24
-    buffer[offset + 2] = keys.length >> 16
-    buffer[offset + 3] = keys.length >> 8
-    buffer[offset + 4] = keys.length
+    buffer[offset + 1] = keysLength >> 24
+    buffer[offset + 2] = keysLength >> 16
+    buffer[offset + 3] = keysLength >> 8
+    buffer[offset + 4] = keysLength
   }
 
   _encode (bytes, trace) {
@@ -171,11 +267,11 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
       payload.metadata['*']['runtime-id'] = this.runtimeId
     }
 
-    this._encodeMapPrefix(bytes, payload)
+    this._encodeMapPrefix(bytes, Object.keys(payload).length)
     this._encodeString(bytes, 'version')
     this._encodeNumber(bytes, payload.version)
     this._encodeString(bytes, 'metadata')
-    this._encodeMapPrefix(bytes, payload.metadata)
+    this._encodeMapPrefix(bytes, Object.keys(payload.metadata).length)
     this._encodeString(bytes, '*')
     this._encodeMap(bytes, payload.metadata['*'])
     this._encodeString(bytes, 'events')

package/packages/dd-trace/src/plugin_manager.js

@@ -36,61 +36,77 @@ const collectDisabledPlugins = () => {
 // TODO this must always be a singleton.
 module.exports = class PluginManager {
   constructor (tracer) {
+    this._tracer = tracer
     this._pluginsByName = {}
     this._configsByName = {}
-
-    const _disabledPlugins = collectDisabledPlugins()
-
-    for (const PluginClass of Object.values(plugins)) {
-      /**
-       * disabling the plugin here instead of in `configure` so we don't waste subscriber
-       * resources on a plugin that will eventually be disabled anyways
-       */
-      if (_disabledPlugins.has(PluginClass.name)) {
-        log.debug(`Plugin "${PluginClass.name}" was disabled via configuration option.`)
-        continue
-      }
-      if (typeof PluginClass !== 'function') continue
-      this._pluginsByName[PluginClass.name] = new PluginClass(tracer)
-      this._configsByName[PluginClass.name] = {}
-    }
+    this._disabledPlugins = collectDisabledPlugins()
   }
 
   // like instrumenter.use()
   configurePlugin (name, pluginConfig) {
-    if (!(name in this._pluginsByName)) return
     if (typeof pluginConfig === 'boolean') {
       pluginConfig = { enabled: pluginConfig }
     }
+    if (!pluginConfig) {
+      pluginConfig = { enabled: true }
+    }
 
-    const config = {
+    this._configsByName[name] = {
       ...this._configsByName[name],
       ...pluginConfig
     }
 
-    this._pluginsByName[name].configure(getConfig(name, config))
+    if (this._pluginsByName[name]) {
+      this._pluginsByName[name].configure(getConfig(name, this._configsByName[name]))
+    }
   }
 
   // like instrumenter.enable()
-  configure (config) {
-    const { logInjection, serviceMapping } = config
-
-    if (config.plugins !== false) {
-      for (const name in this._pluginsByName) {
-        const pluginConfig = {
-          ...this._configsByName[name],
-          logInjection
-        }
-        if (serviceMapping && serviceMapping[name]) {
-          pluginConfig.service = serviceMapping[name]
-        }
-        this.configurePlugin(name, pluginConfig)
+  configure (config = {}) {
+    const { logInjection, serviceMapping, experimental } = config
+
+    for (const PluginClass of Object.values(plugins)) {
+      const name = PluginClass.name
+
+      if (this._disabledPlugins.has(name)) {
+        log.debug(`Plugin "${name}" was disabled via configuration option.`)
+        continue
+      }
+
+      if (typeof PluginClass !== 'function') continue
+
+      this._pluginsByName[name] = new PluginClass(this._tracer)
+
+      if (config.plugins === false) continue
+
+      const pluginConfig = {
+        ...this._configsByName[name]
+      }
+
+      if (logInjection !== undefined) {
+        pluginConfig.logInjection = logInjection
+      }
+
+      // TODO: update so that it's available for every CI Visibility's plugin
+      if (name === 'mocha') {
+        pluginConfig.isAgentlessEnabled = experimental && experimental.exporter === 'datadog'
       }
+
+      if (serviceMapping && serviceMapping[name]) {
+        pluginConfig.service = serviceMapping[name]
+      }
+
+      this.configurePlugin(name, pluginConfig)
     }
   }
 
   // This is basically just for testing. like intrumenter.disable()
   destroy () {
-    for (const name in this._pluginsByName) this._pluginsByName[name].configure({ enabled: false })
+    for (const name in this._pluginsByName) {
+      this._pluginsByName[name].configure({ enabled: false })
+    }
+
+    this._pluginsByName = {}
+    this._configsByName = {}
   }
 }

package/packages/dd-trace/src/plugins/util/test.js

@@ -36,6 +36,9 @@ const TEST_IS_RUM_ACTIVE = 'test.is_rum_active'
 const TEST_CODE_OWNERS = 'test.codeowners'
 const TEST_SOURCE_FILE = 'test.source.file'
 const LIBRARY_VERSION = 'library_version'
+const TEST_COMMAND = 'test.command'
+const TEST_SESSION_ID = 'test_session_id'
+const TEST_SUITE_ID = 'test_suite_id'
 
 const ERROR_TYPE = 'error.type'
 const ERROR_MESSAGE = 'error.msg'
@@ -70,7 +73,12 @@ module.exports = {
   getTestSuitePath,
   getCodeOwnersFileEntries,
   getCodeOwnersForFilename,
-  getTestCommonTags
+  getTestCommonTags,
+  getTestSessionCommonTags,
+  getTestSuiteCommonTags,
+  TEST_COMMAND,
+  TEST_SESSION_ID,
+  TEST_SUITE_ID
 }
 
 function getTestEnvironmentMetadata (testFramework, config) {
@@ -225,3 +233,26 @@ function getCodeOwnersForFilename (filename, entries) {
   }
   return null
 }
+
+function getTestSessionCommonTags (command, version) {
+  return {
+    [SPAN_TYPE]: 'test_session_end',
+    [TEST_TYPE]: 'test',
+    [RESOURCE_NAME]: `test_session.${command}`,
+    [TEST_FRAMEWORK_VERSION]: version,
+    [LIBRARY_VERSION]: ddTraceVersion,
+    [TEST_COMMAND]: command
+  }
+}
+
+function getTestSuiteCommonTags (command, version, testSuite) {
+  return {
+    [SPAN_TYPE]: 'test_suite_end',
+    [TEST_TYPE]: 'test',
+    [RESOURCE_NAME]: `test_suite.${testSuite}`,
+    [TEST_FRAMEWORK_VERSION]: version,
+    [LIBRARY_VERSION]: ddTraceVersion,
+    [TEST_SUITE]: testSuite,
+    [TEST_COMMAND]: command
+  }
+}

package/packages/dd-trace/src/plugins/util/web.js

@@ -259,9 +259,9 @@ const web = {
     const context = contexts.get(req)
     const span = context.span
     const error = context.error
-    const hasMiddlewareError = span.context()._tags['error'] || span.context()._tags['error.msg']
+    const hasExistingError = span.context()._tags['error'] || span.context()._tags['error.msg']
 
-    if (!hasMiddlewareError && !context.config.validateStatus(statusCode)) {
+    if (!hasExistingError && !context.config.validateStatus(statusCode)) {
       span.setTag(ERROR, error || true)
     }
   },
@@ -270,7 +270,10 @@ const web = {
   addError (req, error) {
     if (error instanceof Error) {
       const context = contexts.get(req)
-      context.error = context.error || error
+
+      if (context) {
+        context.error = error
+      }
     }
   },
 
@@ -298,6 +301,23 @@ const web = {
     context.span.finish()
     context.finished = true
   },
+
+  finishAll (context) {
+    const { req, res } = context
+
+    for (const beforeEnd of context.beforeEnd) {
+      beforeEnd()
+    }
+
+    web.finishMiddleware(context)
+
+    if (incomingHttpRequestEnd.hasSubscribers) {
+      incomingHttpRequestEnd.publish({ req, res })
+    }
+
+    web.finishSpan(context)
+  },
+
   wrapWriteHead (context) {
     const { req, res } = context
     const writeHead = res.writeHead
@@ -318,21 +338,9 @@ const web = {
   },
   wrapRes (context, req, res, end) {
     return function () {
-      for (const beforeEnd of context.beforeEnd) {
-        beforeEnd()
-      }
-
-      web.finishMiddleware(context)
-
-      if (incomingHttpRequestEnd.hasSubscribers) {
-        incomingHttpRequestEnd.publish({ req, res })
-      }
-
-      const returnValue = end.apply(res, arguments)
-
-      web.finishSpan(context)
+      web.finishAll(context)
 
-      return returnValue
+      return end.apply(res, arguments)
     }
   },
   wrapEnd (context) {

package/packages/dd-trace/src/profiling/config.js

@@ -6,6 +6,7 @@ const { URL } = require('url')
 const { AgentExporter } = require('./exporters/agent')
 const { FileExporter } = require('./exporters/file')
 const { ConsoleLogger } = require('./loggers/console')
+const CpuProfiler = require('./profilers/cpu')
 const WallProfiler = require('./profilers/wall')
 const SpaceProfiler = require('./profilers/space')
 const { tagger } = require('./tagger')
@@ -13,6 +14,7 @@ const { tagger } = require('./tagger')
 const {
   DD_PROFILING_ENABLED,
   DD_PROFILING_PROFILERS,
+  DD_PROFILING_ENDPOINT_COLLECTION_ENABLED,
   DD_ENV,
   DD_TAGS,
   DD_SERVICE,
@@ -37,6 +39,8 @@ class Config {
       DD_PROFILING_UPLOAD_TIMEOUT, 60 * 1000)
     const sourceMap = coalesce(options.sourceMap,
       DD_PROFILING_SOURCE_MAP, true)
+    const endpointCollection = coalesce(options.endpointCollection,
+      DD_PROFILING_ENDPOINT_COLLECTION_ENABLED, false)
 
     this.enabled = String(enabled) !== 'false'
     this.service = service
@@ -53,6 +57,7 @@ class Config {
     this.flushInterval = flushInterval
     this.uploadTimeout = uploadTimeout
     this.sourceMap = sourceMap
+    this.endpointCollection = endpointCollection
 
     const hostname = coalesce(options.hostname, DD_AGENT_HOST, 'localhost')
     const port = coalesce(options.port, DD_TRACE_AGENT_PORT, 8126)
@@ -64,8 +69,8 @@ class Config {
     ], this)
 
     const profilers = coalesce(options.profilers, DD_PROFILING_PROFILERS, [
-      new WallProfiler(),
-      new SpaceProfiler()
+      new WallProfiler(this),
+      new SpaceProfiler(this)
     ])
 
     this.profilers = ensureProfilers(profilers, this)
@@ -100,10 +105,13 @@ function ensureExporters (exporters, options) {
 
 function getProfiler (name, options) {
   switch (name) {
+    case 'cpu':
     case 'wall':
       return new WallProfiler(options)
     case 'space':
       return new SpaceProfiler(options)
+    case 'cpu-experimental':
+      return new CpuProfiler(options)
     default:
       options.logger.error(`Unknown profiler "${name}"`)
   }

package/packages/dd-trace/src/profiling/exporters/agent.js

@@ -2,7 +2,7 @@
 
 const retry = require('retry')
 const { request } = require('http')
-const FormData = require('form-data')
+const FormData = require('./form-data')
 
 // TODO: avoid using dd-trace internals. Make this a separate module?
 const docker = require('../../exporters/common/docker')
@@ -22,7 +22,6 @@ function sendRequest (options, form, callback) {
   })
   req.on('error', callback)
   if (form) form.pipe(req)
-  req.end()
 }
 
 function getBody (stream, callback) {

package/packages/dd-trace/src/profiling/exporters/form-data.js

@@ -0,0 +1,53 @@
+'use strict'
+
+const { Readable } = require('stream')
+const id = require('../../id')
+
+class FormData extends Readable {
+  constructor () {
+    super()
+
+    this._boundary = id().toString()
+    this._data = []
+  }
+
+  append (key, value, options = {}) {
+    this._appendBoundary()
+
+    if (options.filename) {
+      this._appendFile(key, value, options)
+    } else {
+      this._appendMetadata(key, value, options)
+    }
+  }
+
+  getHeaders () {
+    return { 'Content-Type': 'multipart/form-data; boundary=' + this._boundary }
+  }
+
+  _appendBoundary () {
+    this._data.push(`--${this._boundary}\r\n`)
+  }
+
+  _appendMetadata (key, value) {
+    this._data.push(`Content-Disposition: form-data; name="${key}"\r\n\r\n${value}\r\n`)
+  }
+
+  _appendFile (key, value, { filename, contentType = 'application/octet-stream' }) {
+    this._data.push(`Content-Disposition: form-data; name="${key}"; filename="${filename}"\r\n`)
+    this._data.push(`Content-Type: ${contentType}\r\n\r\n`)
+    this._data.push(value)
+    this._data.push('\r\n')
+  }
+
+  _read () {
+    this.push(this._data.shift())
+
+    if (this._data.length === 0) {
+      this.push(`--${this._boundary}--\r\n`)
+      this.push(null)
+    }
+  }
+}
+
+module.exports = FormData

package/packages/dd-trace/src/profiling/index.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { Profiler } = require('./profiler')
+const CpuProfiler = require('./profilers/cpu')
 const WallProfiler = require('./profilers/wall')
 const SpaceProfiler = require('./profilers/space')
 const { AgentExporter } = require('./exporters/agent')
@@ -13,6 +14,7 @@ module.exports = {
   profiler,
   AgentExporter,
   FileExporter,
+  CpuProfiler,
   WallProfiler,
   SpaceProfiler,
   ConsoleLogger

package/packages/dd-trace/src/profiling/profiler.js

@@ -91,7 +91,12 @@ class Profiler extends EventEmitter {
         if (!profile) continue
 
         profiles[profiler.type] = await profiler.encode(profile)
-        this._logger.debug(`Collected ${profiler.type} profile: ` + JSON.stringify(profile))
+        this._logger.debug(() => {
+          const profileJson = JSON.stringify(profile, (key, value) => {
+            return typeof value === 'bigint' ? value.toString() : value
+          })
+          return `Collected ${profiler.type} profile: ` + profileJson
+        })
       }
 
       this._capture(this._config.flushInterval)