dd-trace 3.12.1 → 3.15.0

package/packages/dd-trace/src/encode/0.4.js

@@ -14,7 +14,7 @@ float64Array[0] = -1
 const bigEndian = uInt8Float64Array[7] === 0
 
 function formatSpan (span) {
-  return normalizeSpan(truncateSpan(span))
+  return normalizeSpan(truncateSpan(span, false))
 }
 
 class AgentEncoder {

package/packages/dd-trace/src/encode/0.5.js

@@ -7,7 +7,7 @@ const ARRAY_OF_TWO = 0x92
 const ARRAY_OF_TWELVE = 0x9c
 
 function formatSpan (span) {
-  return normalizeSpan(truncateSpan(span))
+  return normalizeSpan(truncateSpan(span, false))
 }
 
 class AgentEncoder extends BaseEncoder {

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -5,9 +5,10 @@ const { version: ddTraceVersion } = require('../../../../package.json')
 const id = require('../../../dd-trace/src/id')
 const ENCODING_VERSION = 1
 
-const ALLOWED_CONTENT_TYPES = ['test_session_end', 'test_suite_end', 'test']
+const ALLOWED_CONTENT_TYPES = ['test_session_end', 'test_module_end', 'test_suite_end', 'test']
 
-const TEST_SUITE_KEYS_LENGTH = 11
+const TEST_SUITE_KEYS_LENGTH = 12
+const TEST_MODULE_KEYS_LENGTH = 11
 const TEST_SESSION_KEYS_LENGTH = 10
 
 const INTAKE_SOFT_LIMIT = 2 * 1024 * 1024 // 2MB
@@ -46,6 +47,9 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this._encodeString(bytes, 'test_session_id')
     this._encodeId(bytes, content.trace_id)
 
+    this._encodeString(bytes, 'test_module_id')
+    this._encodeId(bytes, content.parent_id)
+
     this._encodeString(bytes, 'test_suite_id')
     this._encodeId(bytes, content.span_id)
 
@@ -67,6 +71,35 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this._encodeMap(bytes, content.metrics)
   }
 
+  _encodeTestModule (bytes, content) {
+    this._encodeMapPrefix(bytes, TEST_MODULE_KEYS_LENGTH)
+    this._encodeString(bytes, 'type')
+    this._encodeString(bytes, content.type)
+
+    this._encodeString(bytes, 'test_session_id')
+    this._encodeId(bytes, content.trace_id)
+
+    this._encodeString(bytes, 'test_module_id')
+    this._encodeId(bytes, content.span_id)
+
+    this._encodeString(bytes, 'error')
+    this._encodeNumber(bytes, content.error)
+    this._encodeString(bytes, 'name')
+    this._encodeString(bytes, content.name)
+    this._encodeString(bytes, 'service')
+    this._encodeString(bytes, content.service)
+    this._encodeString(bytes, 'resource')
+    this._encodeString(bytes, content.resource)
+    this._encodeString(bytes, 'start')
+    this._encodeNumber(bytes, content.start)
+    this._encodeString(bytes, 'duration')
+    this._encodeNumber(bytes, content.duration)
+    this._encodeString(bytes, 'meta')
+    this._encodeMap(bytes, content.meta)
+    this._encodeString(bytes, 'metrics')
+    this._encodeMap(bytes, content.metrics)
+  }
+
   _encodeTestSession (bytes, content) {
     this._encodeMapPrefix(bytes, TEST_SESSION_KEYS_LENGTH)
     this._encodeString(bytes, 'type')
@@ -95,8 +128,9 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
 
   _encodeEventContent (bytes, content) {
     const keysLength = Object.keys(content).length
+
     if (content.meta.test_session_id) {
-      this._encodeMapPrefix(bytes, keysLength + 2)
+      this._encodeMapPrefix(bytes, keysLength + 3)
     } else {
       this._encodeMapPrefix(bytes, keysLength)
     }
@@ -137,6 +171,10 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
       this._encodeId(bytes, id(content.meta.test_session_id, 10))
       delete content.meta.test_session_id
 
+      this._encodeString(bytes, 'test_module_id')
+      this._encodeId(bytes, id(content.meta.test_module_id, 10))
+      delete content.meta.test_module_id
+
       this._encodeString(bytes, 'test_suite_id')
       this._encodeId(bytes, id(content.meta.test_suite_id, 10))
       delete content.meta.test_suite_id
@@ -160,6 +198,8 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
       this._encodeEventContent(bytes, event.content)
     } else if (event.type === 'test_suite_end') {
       this._encodeTestSuite(bytes, event.content)
+    } else if (event.type === 'test_module_end') {
+      this._encodeTestModule(bytes, event.content)
     } else if (event.type === 'test_session_end') {
       this._encodeTestSession(bytes, event.content)
     }
@@ -200,7 +240,7 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     const rawEvents = trace.map(formatSpan)
 
     const testSessionEvents = rawEvents.filter(
-      event => event.type === 'test_session_end' || event.type === 'test_suite_end'
+      event => event.type === 'test_session_end' || event.type === 'test_suite_end' || event.type === 'test_module_end'
     )
 
     const isTestSessionTrace = !!testSessionEvents.length

package/packages/dd-trace/src/encode/coverage-ci-visibility.js

@@ -1,56 +1,35 @@
 'use strict'
 const { AgentEncoder } = require('./0.4')
 const Chunk = require('./chunk')
-const log = require('../log')
 
 const FormData = require('../exporters/common/form-data')
 
-const COVERAGE_PAYLOAD_VERSION = 1
-const COVERAGE_KEYS_LENGTH = 4
-const MAXIMUM_NUM_COVERAGE_FILES = 100
+const COVERAGE_PAYLOAD_VERSION = 2
+const COVERAGE_KEYS_LENGTH = 2
 
 class CoverageCIVisibilityEncoder extends AgentEncoder {
   constructor () {
     super(...arguments)
     this._coverageBytes = new Chunk()
     this.form = new FormData()
-    this.fileIndex = 1
+    this._coveragesCount = 0
     this.reset()
   }
 
   count () {
-    return this.fileIndex - 1
+    return this._coveragesCount
   }
 
   encode (coverage) {
-    const bytes = this._coverageBytes
-    const coverageBuffer = this.encodeCodeCoverage(bytes, coverage)
-    const coverageFilename = `coverage${this.fileIndex++}`
-
-    this.form.append(
-      coverageFilename,
-      coverageBuffer,
-      {
-        filename: `${coverageFilename}.msgpack`,
-        contentType: 'application/msgpack'
-      }
-    )
-
-    if (this.fileIndex === MAXIMUM_NUM_COVERAGE_FILES) {
-      log.debug('Coverage buffer reached the limit, flushing')
-      this._writer.flush()
-    }
-
-    this.reset()
+    this._coveragesCount++
+    this.encodeCodeCoverage(this._coverageBytes, coverage)
   }
 
   encodeCodeCoverage (bytes, coverage) {
-    this._encodeMapPrefix(bytes, COVERAGE_KEYS_LENGTH)
-    this._encodeString(bytes, 'version')
-    this._encodeInteger(bytes, COVERAGE_PAYLOAD_VERSION)
-    this._encodeString(bytes, 'trace_id')
+    this._encodeMapPrefix(bytes, 3)
+    this._encodeString(bytes, 'test_session_id')
     this._encodeId(bytes, coverage.traceId)
-    this._encodeString(bytes, 'span_id')
+    this._encodeString(bytes, 'test_suite_id')
     this._encodeId(bytes, coverage.spanId)
     this._encodeString(bytes, 'files')
     this._encodeArrayPrefix(bytes, coverage.files)
@@ -59,12 +38,6 @@ class CoverageCIVisibilityEncoder extends AgentEncoder {
       this._encodeString(bytes, 'filename')
       this._encodeString(bytes, filename)
     }
-    const traceSize = bytes.length
-    const buffer = Buffer.allocUnsafe(traceSize)
-
-    bytes.buffer.copy(buffer, 0, 0, bytes.length)
-
-    return buffer
   }
 
   reset () {
@@ -72,9 +45,51 @@ class CoverageCIVisibilityEncoder extends AgentEncoder {
     if (this._coverageBytes) {
       this._coverageBytes.length = 0
     }
+    this._coveragesCount = 0
+    this._encodePayloadStart(this._coverageBytes)
+  }
+
+  _encodePayloadStart (bytes) {
+    const payload = {
+      version: COVERAGE_PAYLOAD_VERSION,
+      coverages: []
+    }
+    this._encodeMapPrefix(bytes, COVERAGE_KEYS_LENGTH)
+    this._encodeString(bytes, 'version')
+    this._encodeInteger(bytes, payload.version)
+    this._encodeString(bytes, 'coverages')
+    // Get offset of the coverages list to update the length of the array when calling `makePayload`
+    this._coveragesOffset = bytes.length
+    bytes.reserve(5)
+    bytes.length += 5
   }
 
   makePayload () {
+    const bytes = this._coverageBytes
+
+    const coveragesOffset = this._coveragesOffset
+    const coveragesCount = this._coveragesCount
+
+    // update with number of coverages
+    bytes.buffer[coveragesOffset] = 0xdd
+    bytes.buffer[coveragesOffset + 1] = coveragesCount >> 24
+    bytes.buffer[coveragesOffset + 2] = coveragesCount >> 16
+    bytes.buffer[coveragesOffset + 3] = coveragesCount >> 8
+    bytes.buffer[coveragesOffset + 4] = coveragesCount
+
+    const traceSize = bytes.length
+    const buffer = Buffer.allocUnsafe(traceSize)
+
+    bytes.buffer.copy(buffer, 0, 0, bytes.length)
+
+    this.form.append(
+      'coverage1',
+      buffer,
+      {
+        filename: `coverage1.msgpack`,
+        contentType: 'application/msgpack'
+      }
+    )
     this.form.append(
       'event',
       // The intake requires a populated dictionary here. Simply having {} is not valid.
@@ -86,7 +101,7 @@ class CoverageCIVisibilityEncoder extends AgentEncoder {
     const form = this.form
 
     this.form = new FormData()
-    this.fileIndex = 1
+    this.reset()
 
     return form
   }

package/packages/dd-trace/src/encode/tags-processors.js

@@ -38,11 +38,12 @@ function truncateToLength (value, maxLength) {
   return value
 }
 
-function truncateSpan (span) {
+// normally the agent truncates the resource and parses it in certain scenarios (e.g. SQL Queries)
+function truncateSpan (span, shouldTruncateResourceName = true) {
   return fromEntries(Object.entries(span).map(([key, value]) => {
     switch (key) {
       case 'resource':
-        return ['resource', truncateToLength(value, MAX_RESOURCE_NAME_LENGTH)]
+        return ['resource', shouldTruncateResourceName ? truncateToLength(value, MAX_RESOURCE_NAME_LENGTH) : value]
       case 'meta':
         return ['meta', fromEntries(Object.entries(value).map(([metaKey, metaValue]) =>
           [truncateToLength(metaKey, MAX_META_KEY_LENGTH), truncateToLength(metaValue, MAX_META_VALUE_LENGTH)]

package/packages/dd-trace/src/exporters/common/request.js

@@ -98,9 +98,16 @@ function request (data, options, callback) {
       if (res.statusCode >= 200 && res.statusCode <= 299) {
         callback(null, responseData, res.statusCode)
       } else {
-        const fullUrl = `${options.url || options.hostname || `localhost:${options.port}`}${options.path}`
-        // eslint-disable-next-line
-        let errorMessage = `Error from ${fullUrl}: ${res.statusCode} ${http.STATUS_CODES[res.statusCode]}.`
+        let errorMessage = ''
+        try {
+          const fullUrl = new URL(
+            options.path,
+            options.url || options.hostname || `http://localhost:${options.port}`
+          ).href
+          errorMessage = `Error from ${fullUrl}: ${res.statusCode} ${http.STATUS_CODES[res.statusCode]}.`
+        } catch (e) {
+          // ignore error
+        }
         if (responseData) {
           errorMessage += ` Response from the endpoint: "${responseData}"`
         }

package/packages/dd-trace/src/lambda/handler.js

@@ -22,15 +22,14 @@ let __lambdaTimeout
  * @param {*} context AWS Lambda context object.
  */
 function checkTimeout (context) {
-  let remainingTimeInMillis = context.getRemainingTimeInMillis()
-  const apmFlushDeadline = parseInt(process.env.DD_APM_FLUSH_DEADLINE)
-  if (apmFlushDeadline && apmFlushDeadline <= remainingTimeInMillis) {
-    remainingTimeInMillis = apmFlushDeadline
-  }
+  const remainingTimeInMillis = context.getRemainingTimeInMillis()
+
+  let apmFlushDeadline = parseInt(process.env.DD_APM_FLUSH_DEADLINE_MILLISECONDS) || 100
+  apmFlushDeadline = apmFlushDeadline < 0 ? 100 : apmFlushDeadline
 
   __lambdaTimeout = setTimeout(() => {
     timeoutChannel.publish(undefined)
-  }, remainingTimeInMillis - 50)
+  }, remainingTimeInMillis - apmFlushDeadline)
 }
 
 /**

package/packages/dd-trace/src/log/channels.js

@@ -0,0 +1,47 @@
+'use strict'
+
+const dc = require('diagnostics_channel')
+
+const Level = {
+  Debug: 'debug',
+  Info: 'info',
+  Warn: 'warn',
+  Error: 'error'
+}
+
+const defaultLevel = Level.Debug
+
+class LogChannel extends dc.Channel {
+  constructor (name, logLevel) {
+    super(`datadog:log:${name}`)
+    this.logLevel = logLevel
+  }
+}
+
+// based on: https://github.com/trentm/node-bunyan#levels
+const logChannels = {
+  [Level.Debug]: new LogChannel(Level.Debug, 20),
+  [Level.Info]: new LogChannel(Level.Info, 30),
+  [Level.Warn]: new LogChannel(Level.Warn, 40),
+  [Level.Error]: new LogChannel(Level.Error, 50)
+}
+
+function getChannelLogLevel (level) {
+  let logChannel
+  if (level && typeof level === 'string') {
+    logChannel = logChannels[level.toLowerCase().trim()] || logChannels[defaultLevel]
+  } else {
+    logChannel = logChannels[defaultLevel]
+  }
+  return logChannel.logLevel
+}
+
+module.exports = {
+  Level,
+  getChannelLogLevel,
+
+  debugChannel: logChannels[Level.Debug],
+  infoChannel: logChannels[Level.Info],
+  warnChannel: logChannels[Level.Warn],
+  errorChannel: logChannels[Level.Error]
+}

package/packages/dd-trace/src/log/index.js

@@ -0,0 +1,79 @@
+'use strict'
+
+const { debugChannel, infoChannel, warnChannel, errorChannel } = require('./channels')
+const logWriter = require('./writer')
+
+const memoize = func => {
+  const cache = {}
+  const memoized = function (key) {
+    if (!cache[key]) {
+      cache[key] = func.apply(this, arguments)
+    }
+
+    return cache[key]
+  }
+
+  return memoized
+}
+
+function processMsg (msg) {
+  return typeof msg === 'function' ? msg() : msg
+}
+
+const log = {
+  use (logger) {
+    logWriter.use(logger)
+    return this
+  },
+
+  toggle (enabled, logLevel) {
+    logWriter.toggle(enabled, logLevel)
+    return this
+  },
+
+  reset () {
+    logWriter.reset()
+    this._deprecate = memoize((code, message) => {
+      errorChannel.publish(message)
+      return true
+    })
+
+    return this
+  },
+
+  debug (message) {
+    if (debugChannel.hasSubscribers) {
+      debugChannel.publish(processMsg(message))
+    }
+    return this
+  },
+
+  info (message) {
+    if (infoChannel.hasSubscribers) {
+      infoChannel.publish(processMsg(message))
+    }
+    return this
+  },
+
+  warn (message) {
+    if (warnChannel.hasSubscribers) {
+      warnChannel.publish(processMsg(message))
+    }
+    return this
+  },
+
+  error (err) {
+    if (errorChannel.hasSubscribers) {
+      errorChannel.publish(processMsg(err))
+    }
+    return this
+  },
+
+  deprecate (code, message) {
+    return this._deprecate(code, message)
+  }
+}
+
+log.reset()
+
+module.exports = log

package/packages/dd-trace/src/log/writer.js

@@ -0,0 +1,124 @@
+'use strict'
+
+const { storage } = require('../../../datadog-core')
+const { getChannelLogLevel, debugChannel, infoChannel, warnChannel, errorChannel } = require('./channels')
+
+const defaultLogger = {
+  debug: msg => console.debug(msg), /* eslint-disable-line no-console */
+  info: msg => console.info(msg), /* eslint-disable-line no-console */
+  warn: msg => console.warn(msg), /* eslint-disable-line no-console */
+  error: msg => console.error(msg) /* eslint-disable-line no-console */
+}
+
+let enabled = false
+let logger = defaultLogger
+let logLevel = getChannelLogLevel()
+
+function withNoop (fn) {
+  const store = storage.getStore()
+
+  storage.enterWith({ noop: true })
+  fn()
+  storage.enterWith(store)
+}
+
+function unsubscribeAll () {
+  if (debugChannel.hasSubscribers) {
+    debugChannel.unsubscribe(onDebug)
+  }
+  if (infoChannel.hasSubscribers) {
+    infoChannel.unsubscribe(onInfo)
+  }
+  if (warnChannel.hasSubscribers) {
+    warnChannel.unsubscribe(onWarn)
+  }
+  if (errorChannel.hasSubscribers) {
+    errorChannel.unsubscribe(onError)
+  }
+}
+
+function toggleSubscription (enable) {
+  unsubscribeAll()
+
+  if (enable) {
+    if (debugChannel.logLevel >= logLevel) {
+      debugChannel.subscribe(onDebug)
+    }
+    if (infoChannel.logLevel >= logLevel) {
+      infoChannel.subscribe(onInfo)
+    }
+    if (warnChannel.logLevel >= logLevel) {
+      warnChannel.subscribe(onWarn)
+    }
+    if (errorChannel.logLevel >= logLevel) {
+      errorChannel.subscribe(onError)
+    }
+  }
+}
+
+function toggle (enable, level) {
+  if (level !== undefined) {
+    logLevel = getChannelLogLevel(level)
+  }
+  enabled = enable
+  toggleSubscription(enabled)
+}
+
+function use (newLogger) {
+  if (newLogger && newLogger.debug instanceof Function && newLogger.error instanceof Function) {
+    logger = newLogger
+  }
+}
+
+function reset () {
+  logger = defaultLogger
+  enabled = false
+  logLevel = getChannelLogLevel()
+  toggleSubscription(false)
+}
+
+function onError (err) {
+  if (enabled) error(err)
+}
+
+function onWarn (message) {
+  if (enabled) warn(message)
+}
+
+function onInfo (message) {
+  if (enabled) info(message)
+}
+
+function onDebug (message) {
+  if (enabled) debug(message)
+}
+
+function error (err) {
+  if (typeof err !== 'object' || !err) {
+    err = String(err)
+  } else if (!err.stack) {
+    err = String(err.message || err)
+  }
+
+  if (typeof err === 'string') {
+    err = new Error(err)
+  }
+
+  withNoop(() => logger.error(err))
+}
+
+function warn (message) {
+  if (!logger.warn) return debug(message)
+  withNoop(() => logger.warn(message))
+}
+
+function info (message) {
+  if (!logger.info) return debug(message)
+  withNoop(() => logger.info(message))
+}
+
+function debug (message) {
+  withNoop(() => logger.debug(message))
+}
+
+module.exports = { use, toggle, reset, error, warn, info, debug }

package/packages/dd-trace/src/metrics.js

@@ -8,6 +8,7 @@ const os = require('os')
 const Client = require('./dogstatsd')
 const log = require('./log')
 const Histogram = require('./histogram')
+const { performance } = require('perf_hooks')
 
 const INTERVAL = 10 * 1000
 
@@ -20,6 +21,7 @@ let cpuUsage
 let gauges
 let counters
 let histograms
+let elu
 
 reset()
 
@@ -259,6 +261,21 @@ function captureHistograms () {
   })
 }
 
+/**
+ * Gathers and reports Event Loop Utilization (ELU) since last run
+ *
+ * ELU is a measure of how busy the event loop is, like running JavaScript or
+ * waiting on *Sync functions. The value is between 0 (idle) and 1 (exhausted).
+ *
+ * performance.eventLoopUtilization available in Node.js >= v14.10, >= v12.19, >= v16
+ */
+const captureELU = ('eventLoopUtilization' in performance) ? () => {
+  // if elu is undefined (first run) the measurement is from start of process
+  elu = performance.eventLoopUtilization(elu)
+
+  client.gauge('runtime.node.event_loop.utilization', elu.utilization)
+} : () => {}
+
 function captureCommonMetrics () {
   captureMemoryUsage()
   captureProcess()
@@ -266,6 +283,7 @@ function captureCommonMetrics () {
   captureGauges()
   captureCounters()
   captureHistograms()
+  captureELU()
 }
 
 function captureNativeMetrics () {

package/packages/dd-trace/src/noop/proxy.js

@@ -1,12 +1,15 @@
 'use strict'
 
 const NoopTracer = require('./tracer')
+const NoopAppsecSdk = require('../appsec/sdk/noop')
 
 const noop = new NoopTracer()
+const noopAppsec = new NoopAppsecSdk()
 
 class Tracer {
   constructor () {
     this._tracer = noop
+    this.appsec = noopAppsec
   }
 
   init () {
@@ -68,8 +71,8 @@ class Tracer {
     return this._tracer.getRumData.apply(this._tracer, arguments)
   }
 
-  setUser () {
-    this._tracer.setUser.apply(this._tracer, arguments)
+  setUser (user) {
+    this.appsec.setUser(user)
     return this
   }
 }