dd-trace 3.14.1 → 3.16.0

package/packages/dd-trace/src/appsec/sdk/track_event.js

@@ -3,6 +3,7 @@
 const log = require('../../log')
 const { getRootSpan } = require('./utils')
 const { MANUAL_KEEP } = require('../../../../../ext/tags')
+const { setUserTags } = require('./set_user')
 
 function trackUserLoginSuccessEvent (tracer, user, metadata) {
   // TODO: better user check here and in _setUser() ?
@@ -17,8 +18,7 @@ function trackUserLoginSuccessEvent (tracer, user, metadata) {
     return
   }
 
-  // TODO: use sdk._setUser(user, rootSpan) (available in User Blocking PR #2710)
-  tracer.setUser(user)
+  setUserTags(user, rootSpan)
 
   trackEvent(tracer, 'users.login.success', metadata, 'trackUserLoginSuccessEvent', rootSpan)
 }

package/packages/dd-trace/src/appsec/sdk/user_blocking.js

@@ -0,0 +1,73 @@
+'use strict'
+
+const { USER_ID } = require('../addresses')
+const Gateway = require('../gateway/engine')
+const { getRootSpan } = require('./utils')
+const { block } = require('../blocking')
+const { storage } = require('../../../../datadog-core')
+const { setUserTags } = require('./set_user')
+const log = require('../../log')
+
+function isUserBlocked (user) {
+  const results = Gateway.propagate({ [USER_ID]: user.id })
+
+  if (!results) {
+    return false
+  }
+
+  for (const entry of results) {
+    if (entry && entry.includes('block')) {
+      return true
+    }
+  }
+
+  return false
+}
+
+function checkUserAndSetUser (tracer, user) {
+  if (!user || !user.id) {
+    log.warn('Invalid user provided to isUserBlocked')
+    return false
+  }
+
+  const rootSpan = getRootSpan(tracer)
+  if (rootSpan) {
+    if (!rootSpan.context()._tags['usr.id']) {
+      setUserTags(user, rootSpan)
+    }
+  } else {
+    log.warn('Root span not available in isUserBlocked')
+  }
+
+  return isUserBlocked(user)
+}
+
+function blockRequest (tracer, req, res) {
+  if (!req || !res) {
+    const store = storage.getStore()
+    if (store) {
+      req = req || store.req
+      res = res || store.res
+    }
+  }
+
+  if (!req || !res) {
+    log.warn('Requests or response object not available in blockRequest')
+    return false
+  }
+
+  const rootSpan = getRootSpan(tracer)
+  if (!rootSpan) {
+    log.warn('Root span not available in blockRequest')
+    return false
+  }
+
+  block(req, res, rootSpan)
+
+  return true
+}
+
+module.exports = {
+  checkUserAndSetUser,
+  blockRequest
+}

package/packages/dd-trace/src/ci-visibility/encode/json-encoder.js

@@ -0,0 +1,27 @@
+'use strict'
+
+class JSONEncoder {
+  constructor () {
+    this.payloads = []
+  }
+
+  encode (payload) {
+    this.payloads.push(payload)
+  }
+
+  count () {
+    return this.payloads.length
+  }
+
+  reset () {
+    this.payloads = []
+  }
+
+  makePayload () {
+    const data = JSON.stringify(this.payloads)
+    this.reset()
+    return data
+  }
+}
+
+module.exports = { JSONEncoder }

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -8,6 +8,19 @@ const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligen
 const log = require('../../log')
 const AgentInfoExporter = require('../../exporters/common/agent-info-exporter')
 
+function getTestConfigurationTags (tags) {
+  if (!tags) {
+    return {}
+  }
+  return Object.keys(tags).reduce((acc, key) => {
+    if (key.startsWith('test.configuration.')) {
+      const [, configKey] = key.split('test.configuration.')
+      acc[configKey] = tags[key]
+    }
+    return acc
+  }, {})
+}
+
 function getIsTestSessionTrace (trace) {
   return trace.some(span =>
     span.type === 'test_session_end' || span.type === 'test_suite_end' || span.type === 'test_module_end'
@@ -95,6 +108,7 @@ class CiVisibilityExporter extends AgentInfoExporter {
         env: this._config.env,
         service: this._config.service,
         isEvpProxy: !!this._isUsingEvpProxy,
+        custom: getTestConfigurationTags(this._config.tags),
         ...testConfiguration
       }
       getSkippableSuitesRequest(configuration, callback)
@@ -119,6 +133,7 @@ class CiVisibilityExporter extends AgentInfoExporter {
         env: this._config.env,
         service: this._config.service,
         isEvpProxy: !!this._isUsingEvpProxy,
+        custom: getTestConfigurationTags(this._config.tags),
         ...testConfiguration
       }
       getItrConfigurationRequest(configuration, (err, itrConfig) => {
@@ -163,23 +178,16 @@ class CiVisibilityExporter extends AgentInfoExporter {
     this._export(trace)
   }
 
-  exportCoverage (coveragePayload) {
+  exportCoverage (formattedCoverage) {
     // Until it's initialized, we just store the coverages as is
     if (!this._isInitialized) {
-      this._coverageBuffer.push(coveragePayload)
+      this._coverageBuffer.push(formattedCoverage)
       return
     }
     if (!this.canReportCodeCoverage()) {
       return
     }
 
-    const { span, coverageFiles } = coveragePayload
-    const formattedCoverage = {
-      traceId: span.context()._traceId,
-      spanId: span.context()._spanId,
-      files: coverageFiles
-    }
-
     this._export(formattedCoverage, this._coverageWriter, '_coverageTimer')
   }
 

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -10,21 +10,23 @@ const {
   getLatestCommits,
   getRepositoryUrl,
   generatePackFilesForCommits,
-  getCommitsToUpload
+  getCommitsToUpload,
+  isShallowRepository,
+  unshallowRepository
 } = require('../../../plugins/util/git')
 
-const isValidSha = (sha) => /[0-9a-f]{40}/.test(sha)
+const isValidSha1 = (sha) => /^[0-9a-f]{40}$/.test(sha)
+const isValidSha256 = (sha) => /^[0-9a-f]{64}$/.test(sha)
 
-function sanitizeCommits (commits) {
+function validateCommits (commits) {
   return commits.map(({ id: commitSha, type }) => {
     if (type !== 'commit') {
       throw new Error('Invalid commit type response')
     }
-    const sanitizedCommit = commitSha.replace(/[^0-9a-f]+/g, '')
-    if (sanitizedCommit !== commitSha || !isValidSha(sanitizedCommit)) {
-      throw new Error('Invalid commit format')
+    if (isValidSha1(commitSha) || isValidSha256(commitSha)) {
+      return commitSha.replace(/[^0-9a-f]+/g, '')
     }
-    return sanitizedCommit
+    throw new Error('Invalid commit format')
   })
 }
 
@@ -82,7 +84,7 @@ function getCommitsToExclude ({ url, isEvpProxy, repositoryUrl }, callback) {
     }
     let commitsToExclude
     try {
-      commitsToExclude = sanitizeCommits(JSON.parse(response).data)
+      commitsToExclude = validateCommits(JSON.parse(response).data)
     } catch (e) {
       return callback(new Error(`Can't parse commits to exclude response: ${e.message}`))
     }
@@ -157,6 +159,10 @@ function sendGitMetadata (url, isEvpProxy, callback) {
     return callback(new Error('Repository URL is empty'))
   }
 
+  if (isShallowRepository()) {
+    unshallowRepository()
+  }
+
   getCommitsToExclude({ url, repositoryUrl, isEvpProxy }, (err, commitsToExclude, headCommit) => {
     if (err) {
       return callback(err)

package/packages/dd-trace/src/ci-visibility/exporters/jest-worker/index.js

@@ -0,0 +1,33 @@
+'use strict'
+
+const Writer = require('./writer')
+const {
+  JEST_WORKER_COVERAGE_PAYLOAD_CODE,
+  JEST_WORKER_TRACE_PAYLOAD_CODE
+} = require('../../../plugins/util/test')
+
+/**
+ * Lightweight exporter whose writers only do simple JSON serialization
+ * of trace and coverage payloads, which they send to the jest main process.
+ */
+class JestWorkerCiVisibilityExporter {
+  constructor () {
+    this._writer = new Writer(JEST_WORKER_TRACE_PAYLOAD_CODE)
+    this._coverageWriter = new Writer(JEST_WORKER_COVERAGE_PAYLOAD_CODE)
+  }
+
+  export (payload) {
+    this._writer.append(payload)
+  }
+
+  exportCoverage (formattedCoverage) {
+    this._coverageWriter.append(formattedCoverage)
+  }
+
+  flush () {
+    this._writer.flush()
+    this._coverageWriter.flush()
+  }
+}
+
+module.exports = JestWorkerCiVisibilityExporter

package/packages/dd-trace/src/ci-visibility/exporters/jest-worker/writer.js

@@ -0,0 +1,37 @@
+'use strict'
+const { JSONEncoder } = require('../../encode/json-encoder')
+
+class Writer {
+  constructor (interprocessCode) {
+    this._encoder = new JSONEncoder()
+    // Code used to identify the type of payload being sent to the main process
+    this._interprocessCode = interprocessCode
+  }
+
+  flush () {
+    const count = this._encoder.count()
+
+    if (count > 0) {
+      const payload = this._encoder.makePayload()
+
+      this._sendPayload(payload)
+    }
+  }
+
+  append (payload) {
+    this._encoder.encode(payload)
+  }
+
+  _sendPayload (data) {
+    // Only available when `child_process` is used for the jest worker.
+    // eslint-disable-next-line
+    // https://github.com/facebook/jest/blob/bb39cb2c617a3334bf18daeca66bd87b7ccab28b/packages/jest-worker/README.md#experimental-worker
+    // If worker_threads is used, this will not work
+    // TODO: make it compatible with worker_threads
+    if (process.send) { // it only works if process.send is available
+      process.send([this._interprocessCode, data])
+    }
+  }
+}
+
+module.exports = Writer

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js

@@ -13,7 +13,8 @@ function getItrConfiguration ({
   osArchitecture,
   runtimeName,
   runtimeVersion,
-  branch
+  branch,
+  custom
 }, done) {
   const options = {
     path: '/api/v2/libraries/tests/services/setting',
@@ -35,9 +36,15 @@ function getItrConfiguration ({
       process.env.DATADOG_APPLICATION_KEY ||
       process.env.DD_APPLICATION_KEY
 
-    if (!apiKey || !appKey) {
-      return done(new Error('App key or API key undefined'))
+    const messagePrefix = 'Request to settings endpoint was not done because Datadog'
+
+    if (!appKey) {
+      return done(new Error(`${messagePrefix} application key is not defined.`))
+    }
+    if (!apiKey) {
+      return done(new Error(`${messagePrefix} API key is not defined.`))
     }
+
     options.headers['dd-api-key'] = apiKey
     options.headers['dd-application-key'] = appKey
   }
@@ -53,7 +60,8 @@ function getItrConfiguration ({
           'os.version': osVersion,
           'os.architecture': osArchitecture,
           'runtime.name': runtimeName,
-          'runtime.version': runtimeVersion
+          'runtime.version': runtimeVersion,
+          custom
         },
         service,
         env,

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js

@@ -11,7 +11,8 @@ function getSkippableSuites ({
   osPlatform,
   osArchitecture,
   runtimeName,
-  runtimeVersion
+  runtimeVersion,
+  custom
 }, done) {
   const options = {
     path: '/api/v2/ci/tests/skippable',
@@ -34,9 +35,15 @@ function getSkippableSuites ({
       process.env.DATADOG_APPLICATION_KEY ||
       process.env.DD_APPLICATION_KEY
 
-    if (!apiKey || !appKey) {
-      return done(new Error('App key or API key undefined'))
+    const messagePrefix = 'Skippable suites were not fetched because Datadog'
+
+    if (!appKey) {
+      return done(new Error(`${messagePrefix} application key is not defined.`))
+    }
+    if (!apiKey) {
+      return done(new Error(`${messagePrefix} API key is not defined.`))
     }
+
     options.headers['dd-api-key'] = apiKey
     options.headers['dd-application-key'] = appKey
   }
@@ -51,7 +58,8 @@ function getSkippableSuites ({
           'os.version': osVersion,
           'os.architecture': osArchitecture,
           'runtime.name': runtimeName,
-          'runtime.version': runtimeVersion
+          'runtime.version': runtimeVersion,
+          custom
         },
         service,
         env,

package/packages/dd-trace/src/config.js

@@ -161,7 +161,7 @@ class Config {
 
     const DD_CIVISIBILITY_ITR_ENABLED = coalesce(
       process.env.DD_CIVISIBILITY_ITR_ENABLED,
-      false
+      true
     )
 
     const DD_SERVICE = options.service ||
@@ -197,6 +197,10 @@ class Config {
       process.env.DD_TRACE_TELEMETRY_ENABLED,
       !process.env.AWS_LAMBDA_FUNCTION_NAME
     )
+    const DD_TELEMETRY_DEBUG_ENABLED = coalesce(
+      process.env.DD_TELEMETRY_DEBUG_ENABLED,
+      false
+    )
     const DD_TRACE_AGENT_PROTOCOL_VERSION = coalesce(
       options.protocolVersion,
       process.env.DD_TRACE_AGENT_PROTOCOL_VERSION,
@@ -352,6 +356,10 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       process.env.DD_IAST_ENABLED,
       false
     )
+    const DD_TELEMETRY_LOG_COLLECTION_ENABLED = coalesce(
+      process.env.DD_TELEMETRY_LOG_COLLECTION_ENABLED,
+      DD_IAST_ENABLED
+    )
 
     const defaultIastRequestSampling = 30
     const iastRequestSampling = coalesce(
@@ -374,9 +382,15 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       2
     )
 
+    const DD_IAST_DEDUPLICATION_ENABLED = coalesce(
+      iastOptions && iastOptions.deduplicationEnabled,
+      process.env.DD_IAST_DEDUPLICATION_ENABLED && isTrue(process.env.DD_IAST_DEDUPLICATION_ENABLED),
+      true
+    )
+
     const DD_CIVISIBILITY_GIT_UPLOAD_ENABLED = coalesce(
       process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED,
-      false
+      true
     )
 
     const ingestion = options.ingestion || {}
@@ -456,7 +470,11 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     this.lookup = options.lookup
     this.startupLogs = isTrue(DD_TRACE_STARTUP_LOGS)
     // Disabled for CI Visibility's agentless
-    this.telemetryEnabled = DD_TRACE_EXPORTER !== 'datadog' && isTrue(DD_TRACE_TELEMETRY_ENABLED)
+    this.telemetry = {
+      enabled: DD_TRACE_EXPORTER !== 'datadog' && isTrue(DD_TRACE_TELEMETRY_ENABLED),
+      logCollection: isTrue(DD_TELEMETRY_LOG_COLLECTION_ENABLED),
+      debug: isTrue(DD_TELEMETRY_DEBUG_ENABLED)
+    }
     this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION
     this.tagsHeaderMaxLength = parseInt(DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH)
     this.appsec = {
@@ -477,14 +495,15 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       enabled: isTrue(DD_IAST_ENABLED),
       requestSampling: DD_IAST_REQUEST_SAMPLING,
       maxConcurrentRequests: DD_IAST_MAX_CONCURRENT_REQUESTS,
-      maxContextOperations: DD_IAST_MAX_CONTEXT_OPERATIONS
+      maxContextOperations: DD_IAST_MAX_CONTEXT_OPERATIONS,
+      deduplicationEnabled: DD_IAST_DEDUPLICATION_ENABLED
     }
 
     this.isCiVisibility = isTrue(DD_IS_CIVISIBILITY)
 
     this.isIntelligentTestRunnerEnabled = this.isCiVisibility && isTrue(DD_CIVISIBILITY_ITR_ENABLED)
     this.isGitUploadEnabled = this.isCiVisibility &&
-      (this.isIntelligentTestRunnerEnabled || isTrue(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED))
+      (this.isIntelligentTestRunnerEnabled && !isFalse(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED))
 
     this.stats = {
       enabled: isTrue(DD_TRACE_STATS_COMPUTATION_ENABLED)

package/packages/dd-trace/src/constants.js

@@ -24,5 +24,6 @@ module.exports = {
   ERROR_TYPE: 'error.type',
   ERROR_MESSAGE: 'error.message',
   ERROR_STACK: 'error.stack',
-  COMPONENT: 'component'
+  COMPONENT: 'component',
+  CLIENT_PORT_KEY: 'network.destination.port'
 }

package/packages/dd-trace/src/datastreams/encoding.js

@@ -0,0 +1,80 @@
+// encodes positive and negative numbers, using zig zag encoding to reduce the size of the variable length encoding.
+// uses high and low part to ensure those parts are under the limit for byte operations in javascript (32 bits)
+// maximum number possible to encode is MAX_SAFE_INTEGER/2 (using zig zag shifts the bits by 1 to the left)
+function encodeVarint (v) {
+  const sign = v >= 0 ? 0 : 1
+  // we leave the least significant bit for the sign.
+  const double = Math.abs(v) * 2
+  if (double > Number.MAX_SAFE_INTEGER) {
+    return undefined
+  }
+  const high = Math.floor(double / 0x100000000)
+  const low = (double & 0xffffffff) | sign
+  return encodeUvarint64(low, high)
+}
+
+// decodes positive and negative numbers, using zig zag encoding to reduce the size of the variable length encoding.
+// uses high and low part to ensure those parts are under the limit for byte operations in javascript (32 bits)
+function decodeVarint (b) {
+  const [low, high] = decodeUvarint64(b)
+  if (low === undefined || high === undefined) {
+    return undefined
+  }
+  const positive = (low & 1) === 0
+  const abs = (low >>> 1) + high * 0x80000000
+  return positive ? abs : -abs
+}
+
+const maxVarLen64 = 9
+
+function encodeUvarint64 (low, high) {
+  const result = new Uint8Array(maxVarLen64)
+  let i = 0
+  // if first byte is 1, the number is negative in javascript, but we want to interpret it as positive
+  while ((high !== 0 || low < 0 || low > 0x80) && i < maxVarLen64 - 1) {
+    result[i] = (low & 0x7f) | 0x80
+    low >>>= 7
+    low |= (high & 0x7f) << 25
+    high >>>= 7
+    i++
+  }
+  result[i] = low & 0x7f
+  return result.slice(0, i + 1)
+}
+
+function decodeUvarint64 (
+  bytes
+) {
+  let low = 0
+  let high = 0
+  let s = 0
+  for (let i = 0; ; i++) {
+    if (bytes.length <= i) {
+      return [undefined, undefined]
+    }
+    const n = bytes[i]
+    if (n < 0x80 || i === maxVarLen64 - 1) {
+      bytes = bytes.slice(i + 1)
+      if (s < 32) {
+        low |= n << s
+      }
+      if (s > 0) {
+        high |= s - 32 > 0 ? n << (s - 32) : n >> (32 - s)
+      }
+      return [low, high]
+    }
+    if (s < 32) {
+      low |= (n & 0x7f) << s
+    }
+    if (s > 0) {
+      high |=
+        s - 32 > 0 ? (n & 0x7f) << (s - 32) : (n & 0x7f) >> (32 - s)
+    }
+    s += 7
+  }
+}
+
+module.exports = {
+  encodeVarint,
+  decodeVarint
+}

package/packages/dd-trace/src/exporter.js

@@ -1,9 +1,5 @@
 'use strict'
 
-const AgentExporter = require('./exporters/agent')
-const LogExporter = require('./exporters/log')
-const AgentlessCiVisibilityExporter = require('./ci-visibility/exporters/agentless')
-const AgentProxyCiVisibilityExporter = require('./ci-visibility/exporters/agent-proxy')
 const exporters = require('../../../ext/exporters')
 const fs = require('fs')
 const constants = require('./constants')
@@ -14,14 +10,16 @@ module.exports = name => {
 
   switch (name) {
     case exporters.LOG:
-      return LogExporter
+      return require('./exporters/log')
     case exporters.AGENT:
-      return AgentExporter
+      return require('./exporters/agent')
     case exporters.DATADOG:
-      return AgentlessCiVisibilityExporter
+      return require('./ci-visibility/exporters/agentless')
     case exporters.AGENT_PROXY:
-      return AgentProxyCiVisibilityExporter
+      return require('./ci-visibility/exporters/agent-proxy')
+    case exporters.JEST_WORKER:
+      return require('./ci-visibility/exporters/jest-worker')
     default:
-      return inAWSLambda && !usingLambdaExtension ? LogExporter : AgentExporter
+      return inAWSLambda && !usingLambdaExtension ? require('./exporters/log') : require('./exporters/agent')
   }
 }

package/packages/dd-trace/src/exporters/common/agents.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const http = require('http')
+const https = require('https')
+const { storage } = require('../../../../datadog-core')
+
+const keepAlive = true
+const maxSockets = 1
+
+function createAgentClass (BaseAgent) {
+  class CustomAgent extends BaseAgent {
+    constructor () {
+      super({ keepAlive, maxSockets })
+    }
+
+    createConnection (...args) {
+      return this._noop(() => super.createConnection(...args))
+    }
+
+    keepSocketAlive (...args) {
+      return this._noop(() => super.keepSocketAlive(...args))
+    }
+
+    reuseSocket (...args) {
+      return this._noop(() => super.reuseSocket(...args))
+    }
+
+    _noop (callback) {
+      return storage.run({ noop: true }, callback)
+    }
+  }
+
+  return CustomAgent
+}
+
+const HttpAgent = createAgentClass(http.Agent)
+const HttpsAgent = createAgentClass(https.Agent)
+
+module.exports = {
+  httpAgent: new HttpAgent(),
+  HttpsAgent: new HttpsAgent()
+}

package/packages/dd-trace/src/exporters/common/docker.js

@@ -2,7 +2,10 @@
 
 const fs = require('fs')
 
-const uuidSource = '[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}'
+// The second part is the PCF / Garden regexp. We currently assume no suffix($) to avoid matching pod UIDs
+// See https://github.com/DataDog/datadog-agent/blob/7.40.x/pkg/util/cgroups/reader.go#L50
+const uuidSource =
+'[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}|[0-9a-f]{8}(?:-[0-9a-f]{4}){4}$'
 const containerSource = '[0-9a-f]{64}'
 const taskSource = '[0-9a-f]{32}-\\d+'
 const entityReg = new RegExp(`.*(${uuidSource}|${containerSource}|${taskSource})(?:\\.scope)?$`, 'm')

package/packages/dd-trace/src/exporters/common/request.js

@@ -8,14 +8,11 @@ const http = require('http')
 const https = require('https')
 const { parse: urlParse } = require('url')
 const docker = require('./docker')
+const { httpAgent, httpsAgent } = require('./agents')
 const { storage } = require('../../../../datadog-core')
 const log = require('../../log')
 
-const keepAlive = true
-const maxSockets = 1
 const maxActiveRequests = 8
-const httpAgent = new http.Agent({ keepAlive, maxSockets })
-const httpsAgent = new https.Agent({ keepAlive, maxSockets })
 const containerId = docker.id()
 
 let activeRequests = 0