dd-trace 3.15.0 → 3.16.0

package/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js

@@ -4,8 +4,11 @@ const VULNERABILITIES_KEY = 'vulnerabilities'
 const IAST_JSON_TAG_KEY = '_dd.iast.json'
 const VULNERABILITY_HASHES_MAX_SIZE = 1000
 const VULNERABILITY_HASHES = new LRU({ max: VULNERABILITY_HASHES_MAX_SIZE })
+const RESET_VULNERABILITY_CACHE_INTERVAL = 60 * 60 * 1000 // 1 hour
 
 let tracer
+let resetVulnerabilityCacheTimer
+let deduplicationEnabled = true
 
 function createVulnerability (type, evidence, spanId, location) {
   if (type && evidence) {
@@ -164,7 +167,20 @@ function clearCache () { // only for test purposes
   VULNERABILITY_HASHES.clear()
 }
 
+function startClearCacheTimer () {
+  resetVulnerabilityCacheTimer = setInterval(clearCache, RESET_VULNERABILITY_CACHE_INTERVAL)
+  resetVulnerabilityCacheTimer.unref()
+}
+
+function stopClearCacheTimer () {
+  if (resetVulnerabilityCacheTimer) {
+    clearInterval(resetVulnerabilityCacheTimer)
+    resetVulnerabilityCacheTimer = null
+  }
+}
+
 function deduplicateVulnerabilities (vulnerabilities) {
+  if (!deduplicationEnabled) return vulnerabilities
   const deduplicated = vulnerabilities.filter((vulnerability) => {
     const key = `${vulnerability.type}${vulnerability.hash}`
     if (!VULNERABILITY_HASHES.get(key)) {
@@ -176,14 +192,23 @@ function deduplicateVulnerabilities (vulnerabilities) {
   return deduplicated
 }
 
-function setTracer (_tracer) {
+function start (config, _tracer) {
+  deduplicationEnabled = config.iast.deduplicationEnabled
+  if (deduplicationEnabled) {
+    startClearCacheTimer()
+  }
   tracer = _tracer
 }
 
+function stop () {
+  stopClearCacheTimer()
+}
+
 module.exports = {
   createVulnerability,
   addVulnerability,
   sendVulnerabilities,
   clearCache,
-  setTracer
+  start,
+  stop
 }

package/packages/dd-trace/src/ci-visibility/encode/json-encoder.js

@@ -0,0 +1,27 @@
+'use strict'
+
+class JSONEncoder {
+  constructor () {
+    this.payloads = []
+  }
+
+  encode (payload) {
+    this.payloads.push(payload)
+  }
+
+  count () {
+    return this.payloads.length
+  }
+
+  reset () {
+    this.payloads = []
+  }
+
+  makePayload () {
+    const data = JSON.stringify(this.payloads)
+    this.reset()
+    return data
+  }
+}
+
+module.exports = { JSONEncoder }

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -178,23 +178,16 @@ class CiVisibilityExporter extends AgentInfoExporter {
     this._export(trace)
   }
 
-  exportCoverage (coveragePayload) {
+  exportCoverage (formattedCoverage) {
     // Until it's initialized, we just store the coverages as is
     if (!this._isInitialized) {
-      this._coverageBuffer.push(coveragePayload)
+      this._coverageBuffer.push(formattedCoverage)
       return
     }
     if (!this.canReportCodeCoverage()) {
       return
     }
 
-    const { span, coverageFiles } = coveragePayload
-    const formattedCoverage = {
-      traceId: span.context()._traceId,
-      spanId: span.context()._spanId,
-      files: coverageFiles
-    }
-
     this._export(formattedCoverage, this._coverageWriter, '_coverageTimer')
   }
 

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -15,18 +15,18 @@ const {
   unshallowRepository
 } = require('../../../plugins/util/git')
 
-const isValidSha = (sha) => /[0-9a-f]{40}/.test(sha)
+const isValidSha1 = (sha) => /^[0-9a-f]{40}$/.test(sha)
+const isValidSha256 = (sha) => /^[0-9a-f]{64}$/.test(sha)
 
-function sanitizeCommits (commits) {
+function validateCommits (commits) {
   return commits.map(({ id: commitSha, type }) => {
     if (type !== 'commit') {
       throw new Error('Invalid commit type response')
     }
-    const sanitizedCommit = commitSha.replace(/[^0-9a-f]+/g, '')
-    if (sanitizedCommit !== commitSha || !isValidSha(sanitizedCommit)) {
-      throw new Error('Invalid commit format')
+    if (isValidSha1(commitSha) || isValidSha256(commitSha)) {
+      return commitSha.replace(/[^0-9a-f]+/g, '')
     }
-    return sanitizedCommit
+    throw new Error('Invalid commit format')
   })
 }
 
@@ -84,7 +84,7 @@ function getCommitsToExclude ({ url, isEvpProxy, repositoryUrl }, callback) {
     }
     let commitsToExclude
     try {
-      commitsToExclude = sanitizeCommits(JSON.parse(response).data)
+      commitsToExclude = validateCommits(JSON.parse(response).data)
     } catch (e) {
       return callback(new Error(`Can't parse commits to exclude response: ${e.message}`))
     }

package/packages/dd-trace/src/ci-visibility/exporters/jest-worker/index.js

@@ -0,0 +1,33 @@
+'use strict'
+
+const Writer = require('./writer')
+const {
+  JEST_WORKER_COVERAGE_PAYLOAD_CODE,
+  JEST_WORKER_TRACE_PAYLOAD_CODE
+} = require('../../../plugins/util/test')
+
+/**
+ * Lightweight exporter whose writers only do simple JSON serialization
+ * of trace and coverage payloads, which they send to the jest main process.
+ */
+class JestWorkerCiVisibilityExporter {
+  constructor () {
+    this._writer = new Writer(JEST_WORKER_TRACE_PAYLOAD_CODE)
+    this._coverageWriter = new Writer(JEST_WORKER_COVERAGE_PAYLOAD_CODE)
+  }
+
+  export (payload) {
+    this._writer.append(payload)
+  }
+
+  exportCoverage (formattedCoverage) {
+    this._coverageWriter.append(formattedCoverage)
+  }
+
+  flush () {
+    this._writer.flush()
+    this._coverageWriter.flush()
+  }
+}
+
+module.exports = JestWorkerCiVisibilityExporter

package/packages/dd-trace/src/ci-visibility/exporters/jest-worker/writer.js

@@ -0,0 +1,37 @@
+'use strict'
+const { JSONEncoder } = require('../../encode/json-encoder')
+
+class Writer {
+  constructor (interprocessCode) {
+    this._encoder = new JSONEncoder()
+    // Code used to identify the type of payload being sent to the main process
+    this._interprocessCode = interprocessCode
+  }
+
+  flush () {
+    const count = this._encoder.count()
+
+    if (count > 0) {
+      const payload = this._encoder.makePayload()
+
+      this._sendPayload(payload)
+    }
+  }
+
+  append (payload) {
+    this._encoder.encode(payload)
+  }
+
+  _sendPayload (data) {
+    // Only available when `child_process` is used for the jest worker.
+    // eslint-disable-next-line
+    // https://github.com/facebook/jest/blob/bb39cb2c617a3334bf18daeca66bd87b7ccab28b/packages/jest-worker/README.md#experimental-worker
+    // If worker_threads is used, this will not work
+    // TODO: make it compatible with worker_threads
+    if (process.send) { // it only works if process.send is available
+      process.send([this._interprocessCode, data])
+    }
+  }
+}
+
+module.exports = Writer

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js

@@ -36,9 +36,15 @@ function getItrConfiguration ({
       process.env.DATADOG_APPLICATION_KEY ||
       process.env.DD_APPLICATION_KEY
 
-    if (!apiKey || !appKey) {
-      return done(new Error('App key or API key undefined'))
+    const messagePrefix = 'Request to settings endpoint was not done because Datadog'
+
+    if (!appKey) {
+      return done(new Error(`${messagePrefix} application key is not defined.`))
+    }
+    if (!apiKey) {
+      return done(new Error(`${messagePrefix} API key is not defined.`))
     }
+
     options.headers['dd-api-key'] = apiKey
     options.headers['dd-application-key'] = appKey
   }

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js

@@ -35,9 +35,15 @@ function getSkippableSuites ({
       process.env.DATADOG_APPLICATION_KEY ||
       process.env.DD_APPLICATION_KEY
 
-    if (!apiKey || !appKey) {
-      return done(new Error('App key or API key undefined'))
+    const messagePrefix = 'Skippable suites were not fetched because Datadog'
+
+    if (!appKey) {
+      return done(new Error(`${messagePrefix} application key is not defined.`))
+    }
+    if (!apiKey) {
+      return done(new Error(`${messagePrefix} API key is not defined.`))
     }
+
     options.headers['dd-api-key'] = apiKey
     options.headers['dd-application-key'] = appKey
   }

package/packages/dd-trace/src/config.js

@@ -161,7 +161,7 @@ class Config {
 
     const DD_CIVISIBILITY_ITR_ENABLED = coalesce(
       process.env.DD_CIVISIBILITY_ITR_ENABLED,
-      false
+      true
     )
 
     const DD_SERVICE = options.service ||
@@ -197,6 +197,10 @@ class Config {
       process.env.DD_TRACE_TELEMETRY_ENABLED,
       !process.env.AWS_LAMBDA_FUNCTION_NAME
     )
+    const DD_TELEMETRY_DEBUG_ENABLED = coalesce(
+      process.env.DD_TELEMETRY_DEBUG_ENABLED,
+      false
+    )
     const DD_TRACE_AGENT_PROTOCOL_VERSION = coalesce(
       options.protocolVersion,
       process.env.DD_TRACE_AGENT_PROTOCOL_VERSION,
@@ -352,6 +356,10 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       process.env.DD_IAST_ENABLED,
       false
     )
+    const DD_TELEMETRY_LOG_COLLECTION_ENABLED = coalesce(
+      process.env.DD_TELEMETRY_LOG_COLLECTION_ENABLED,
+      DD_IAST_ENABLED
+    )
 
     const defaultIastRequestSampling = 30
     const iastRequestSampling = coalesce(
@@ -374,6 +382,12 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       2
     )
 
+    const DD_IAST_DEDUPLICATION_ENABLED = coalesce(
+      iastOptions && iastOptions.deduplicationEnabled,
+      process.env.DD_IAST_DEDUPLICATION_ENABLED && isTrue(process.env.DD_IAST_DEDUPLICATION_ENABLED),
+      true
+    )
+
     const DD_CIVISIBILITY_GIT_UPLOAD_ENABLED = coalesce(
       process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED,
       true
@@ -456,7 +470,11 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     this.lookup = options.lookup
     this.startupLogs = isTrue(DD_TRACE_STARTUP_LOGS)
     // Disabled for CI Visibility's agentless
-    this.telemetryEnabled = DD_TRACE_EXPORTER !== 'datadog' && isTrue(DD_TRACE_TELEMETRY_ENABLED)
+    this.telemetry = {
+      enabled: DD_TRACE_EXPORTER !== 'datadog' && isTrue(DD_TRACE_TELEMETRY_ENABLED),
+      logCollection: isTrue(DD_TELEMETRY_LOG_COLLECTION_ENABLED),
+      debug: isTrue(DD_TELEMETRY_DEBUG_ENABLED)
+    }
     this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION
     this.tagsHeaderMaxLength = parseInt(DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH)
     this.appsec = {
@@ -477,14 +495,15 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       enabled: isTrue(DD_IAST_ENABLED),
       requestSampling: DD_IAST_REQUEST_SAMPLING,
       maxConcurrentRequests: DD_IAST_MAX_CONCURRENT_REQUESTS,
-      maxContextOperations: DD_IAST_MAX_CONTEXT_OPERATIONS
+      maxContextOperations: DD_IAST_MAX_CONTEXT_OPERATIONS,
+      deduplicationEnabled: DD_IAST_DEDUPLICATION_ENABLED
     }
 
     this.isCiVisibility = isTrue(DD_IS_CIVISIBILITY)
 
     this.isIntelligentTestRunnerEnabled = this.isCiVisibility && isTrue(DD_CIVISIBILITY_ITR_ENABLED)
     this.isGitUploadEnabled = this.isCiVisibility &&
-      (this.isIntelligentTestRunnerEnabled || isTrue(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED))
+      (this.isIntelligentTestRunnerEnabled && !isFalse(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED))
 
     this.stats = {
       enabled: isTrue(DD_TRACE_STATS_COMPUTATION_ENABLED)

package/packages/dd-trace/src/constants.js

@@ -24,5 +24,6 @@ module.exports = {
   ERROR_TYPE: 'error.type',
   ERROR_MESSAGE: 'error.message',
   ERROR_STACK: 'error.stack',
-  COMPONENT: 'component'
+  COMPONENT: 'component',
+  CLIENT_PORT_KEY: 'network.destination.port'
 }

package/packages/dd-trace/src/datastreams/encoding.js

@@ -0,0 +1,80 @@
+// encodes positive and negative numbers, using zig zag encoding to reduce the size of the variable length encoding.
+// uses high and low part to ensure those parts are under the limit for byte operations in javascript (32 bits)
+// maximum number possible to encode is MAX_SAFE_INTEGER/2 (using zig zag shifts the bits by 1 to the left)
+function encodeVarint (v) {
+  const sign = v >= 0 ? 0 : 1
+  // we leave the least significant bit for the sign.
+  const double = Math.abs(v) * 2
+  if (double > Number.MAX_SAFE_INTEGER) {
+    return undefined
+  }
+  const high = Math.floor(double / 0x100000000)
+  const low = (double & 0xffffffff) | sign
+  return encodeUvarint64(low, high)
+}
+
+// decodes positive and negative numbers, using zig zag encoding to reduce the size of the variable length encoding.
+// uses high and low part to ensure those parts are under the limit for byte operations in javascript (32 bits)
+function decodeVarint (b) {
+  const [low, high] = decodeUvarint64(b)
+  if (low === undefined || high === undefined) {
+    return undefined
+  }
+  const positive = (low & 1) === 0
+  const abs = (low >>> 1) + high * 0x80000000
+  return positive ? abs : -abs
+}
+
+const maxVarLen64 = 9
+
+function encodeUvarint64 (low, high) {
+  const result = new Uint8Array(maxVarLen64)
+  let i = 0
+  // if first byte is 1, the number is negative in javascript, but we want to interpret it as positive
+  while ((high !== 0 || low < 0 || low > 0x80) && i < maxVarLen64 - 1) {
+    result[i] = (low & 0x7f) | 0x80
+    low >>>= 7
+    low |= (high & 0x7f) << 25
+    high >>>= 7
+    i++
+  }
+  result[i] = low & 0x7f
+  return result.slice(0, i + 1)
+}
+
+function decodeUvarint64 (
+  bytes
+) {
+  let low = 0
+  let high = 0
+  let s = 0
+  for (let i = 0; ; i++) {
+    if (bytes.length <= i) {
+      return [undefined, undefined]
+    }
+    const n = bytes[i]
+    if (n < 0x80 || i === maxVarLen64 - 1) {
+      bytes = bytes.slice(i + 1)
+      if (s < 32) {
+        low |= n << s
+      }
+      if (s > 0) {
+        high |= s - 32 > 0 ? n << (s - 32) : n >> (32 - s)
+      }
+      return [low, high]
+    }
+    if (s < 32) {
+      low |= (n & 0x7f) << s
+    }
+    if (s > 0) {
+      high |=
+        s - 32 > 0 ? (n & 0x7f) << (s - 32) : (n & 0x7f) >> (32 - s)
+    }
+    s += 7
+  }
+}
+
+module.exports = {
+  encodeVarint,
+  decodeVarint
+}

package/packages/dd-trace/src/exporter.js

@@ -1,9 +1,5 @@
 'use strict'
 
-const AgentExporter = require('./exporters/agent')
-const LogExporter = require('./exporters/log')
-const AgentlessCiVisibilityExporter = require('./ci-visibility/exporters/agentless')
-const AgentProxyCiVisibilityExporter = require('./ci-visibility/exporters/agent-proxy')
 const exporters = require('../../../ext/exporters')
 const fs = require('fs')
 const constants = require('./constants')
@@ -14,14 +10,16 @@ module.exports = name => {
 
   switch (name) {
     case exporters.LOG:
-      return LogExporter
+      return require('./exporters/log')
     case exporters.AGENT:
-      return AgentExporter
+      return require('./exporters/agent')
     case exporters.DATADOG:
-      return AgentlessCiVisibilityExporter
+      return require('./ci-visibility/exporters/agentless')
     case exporters.AGENT_PROXY:
-      return AgentProxyCiVisibilityExporter
+      return require('./ci-visibility/exporters/agent-proxy')
+    case exporters.JEST_WORKER:
+      return require('./ci-visibility/exporters/jest-worker')
     default:
-      return inAWSLambda && !usingLambdaExtension ? LogExporter : AgentExporter
+      return inAWSLambda && !usingLambdaExtension ? require('./exporters/log') : require('./exporters/agent')
   }
 }

package/packages/dd-trace/src/exporters/common/agents.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const http = require('http')
+const https = require('https')
+const { storage } = require('../../../../datadog-core')
+
+const keepAlive = true
+const maxSockets = 1
+
+function createAgentClass (BaseAgent) {
+  class CustomAgent extends BaseAgent {
+    constructor () {
+      super({ keepAlive, maxSockets })
+    }
+
+    createConnection (...args) {
+      return this._noop(() => super.createConnection(...args))
+    }
+
+    keepSocketAlive (...args) {
+      return this._noop(() => super.keepSocketAlive(...args))
+    }
+
+    reuseSocket (...args) {
+      return this._noop(() => super.reuseSocket(...args))
+    }
+
+    _noop (callback) {
+      return storage.run({ noop: true }, callback)
+    }
+  }
+
+  return CustomAgent
+}
+
+const HttpAgent = createAgentClass(http.Agent)
+const HttpsAgent = createAgentClass(https.Agent)
+
+module.exports = {
+  httpAgent: new HttpAgent(),
+  HttpsAgent: new HttpsAgent()
+}

package/packages/dd-trace/src/exporters/common/docker.js

@@ -2,7 +2,10 @@
 
 const fs = require('fs')
 
-const uuidSource = '[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}'
+// The second part is the PCF / Garden regexp. We currently assume no suffix($) to avoid matching pod UIDs
+// See https://github.com/DataDog/datadog-agent/blob/7.40.x/pkg/util/cgroups/reader.go#L50
+const uuidSource =
+'[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}|[0-9a-f]{8}(?:-[0-9a-f]{4}){4}$'
 const containerSource = '[0-9a-f]{64}'
 const taskSource = '[0-9a-f]{32}-\\d+'
 const entityReg = new RegExp(`.*(${uuidSource}|${containerSource}|${taskSource})(?:\\.scope)?$`, 'm')

package/packages/dd-trace/src/exporters/common/request.js

@@ -8,14 +8,11 @@ const http = require('http')
 const https = require('https')
 const { parse: urlParse } = require('url')
 const docker = require('./docker')
+const { httpAgent, httpsAgent } = require('./agents')
 const { storage } = require('../../../../datadog-core')
 const log = require('../../log')
 
-const keepAlive = true
-const maxSockets = 1
 const maxActiveRequests = 8
-const httpAgent = new http.Agent({ keepAlive, maxSockets })
-const httpsAgent = new https.Agent({ keepAlive, maxSockets })
 const containerId = docker.id()
 
 let activeRequests = 0

package/packages/dd-trace/src/lambda/handler.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const log = require('../log')
 const { channel } = require('../../../datadog-instrumentations/src/helpers/instrument')
 const { ERROR_MESSAGE, ERROR_TYPE } = require('../constants')
 const { ImpendingTimeout } = require('./runtime/errors')
@@ -42,13 +43,20 @@ function checkTimeout (context) {
  */
 function crashFlush () {
   const activeSpan = tracer.scope().active()
-  const error = new ImpendingTimeout('Datadog detected an impending timeout')
-  activeSpan.addTags({
-    [ERROR_MESSAGE]: error.message,
-    [ERROR_TYPE]: error.name
-  })
+  if (activeSpan !== null) {
+    const error = new ImpendingTimeout('Datadog detected an impending timeout')
+    activeSpan.addTags({
+      [ERROR_MESSAGE]: error.message,
+      [ERROR_TYPE]: error.name
+    })
+  } else {
+    log.warn('An impending timeout was reached, but no root span was found. No error will be tagged.')
+  }
+
   tracer._processor.killAll()
-  activeSpan.finish()
+  if (activeSpan !== null) {
+    activeSpan.finish()
+  }
 }
 
 /**

package/packages/dd-trace/src/opentracing/span.js

@@ -32,6 +32,11 @@ class DatadogSpan {
     this._processor = processor
     this._prioritySampler = prioritySampler
     this._store = storage.getStore()
+    this._duration = undefined
+
+    // For internal use only. You probably want `context()._name`.
+    // This name property is not updated when the span name changes.
+    // This is necessary for span count metrics.
     this._name = operationName
 
     this._spanContext = this._createContext(parent)

package/packages/dd-trace/src/plugin_manager.js

@@ -28,17 +28,17 @@ loadChannel.subscribe(({ name }) => {
   const Plugin = plugins[name]
 
   if (!Plugin || typeof Plugin !== 'function') return
-  if (!pluginClasses[Plugin.name]) {
-    const envName = `DD_TRACE_${Plugin.name.toUpperCase()}_ENABLED`
+  if (!pluginClasses[Plugin.id]) {
+    const envName = `DD_TRACE_${Plugin.id.toUpperCase()}_ENABLED`
     const enabled = process.env[envName.replace(/[^a-z0-9_]/ig, '_')]
 
     // TODO: remove the need to load the plugin class in order to disable the plugin
-    if (isFalse(enabled) || disabledPlugins.has(Plugin.name)) {
-      log.debug(`Plugin "${Plugin.name}" was disabled via configuration option.`)
+    if (isFalse(enabled) || disabledPlugins.has(Plugin.id)) {
+      log.debug(`Plugin "${Plugin.id}" was disabled via configuration option.`)
 
-      pluginClasses[Plugin.name] = null
+      pluginClasses[Plugin.id] = null
     } else {
-      pluginClasses[Plugin.name] = Plugin
+      pluginClasses[Plugin.id] = Plugin
     }
   }
 })
@@ -56,7 +56,7 @@ module.exports = class PluginManager {
 
       if (!Plugin || typeof Plugin !== 'function') return
 
-      this.loadPlugin(Plugin.name)
+      this.loadPlugin(Plugin.id)
     }
 
     loadChannel.subscribe(this._loadedSubscriber)