dd-trace 3.8.0 → 3.9.0

package/packages/dd-trace/src/appsec/remote_config/manager.js

@@ -0,0 +1,264 @@
+'use strict'
+
+const uuid = require('crypto-randomuuid')
+const { EventEmitter } = require('events')
+const Scheduler = require('./scheduler')
+const tracerVersion = require('../../../../../package.json').version
+const request = require('../../exporters/common/request')
+const log = require('../../log')
+
+const clientId = uuid()
+
+const POLL_INTERVAL = 5e3
+const DEFAULT_CAPABILITY = Buffer.alloc(1).toString('base64') // 0x00
+
+// There MUST NOT exist separate instances of RC clients in a tracer making separate ClientGetConfigsRequest
+// with their own separated Client.ClientState.
+class RemoteConfigManager extends EventEmitter {
+  constructor (config) {
+    super()
+
+    this.scheduler = new Scheduler((cb) => this.poll(cb), POLL_INTERVAL)
+
+    this.requestOptions = {
+      url: config.url,
+      hostname: config.hostname,
+      port: config.port,
+      method: 'POST',
+      path: '/v0.7/config'
+    }
+
+    this.state = {
+      client: {
+        state: { // updated by `parseConfig()`
+          root_version: 1,
+          targets_version: 0,
+          config_states: [],
+          has_error: false,
+          error: '',
+          backend_client_state: ''
+        },
+        id: clientId,
+        products: [], // updated by `updateProducts()`
+        is_tracer: true,
+        client_tracer: {
+          runtime_id: config.tags['runtime-id'],
+          language: 'node',
+          tracer_version: tracerVersion,
+          service: config.service,
+          env: config.env,
+          app_version: config.version
+        },
+        capabilities: DEFAULT_CAPABILITY // updated by `updateCapabilities()`
+      },
+      cached_target_files: [] // updated by `parseConfig()`
+    }
+
+    this.appliedConfigs = new Map()
+  }
+
+  updateCapabilities (mask, value) {
+    const hex = Buffer.from(this.state.client.capabilities, 'base64').toString('hex')
+
+    let num = BigInt(`0x${hex}`)
+
+    if (value) {
+      num |= mask
+    } else {
+      num &= ~mask
+    }
+
+    let str = num.toString(16)
+
+    if (str.length % 2) str = `0${str}`
+
+    this.state.client.capabilities = Buffer.from(str, 'hex').toString('base64')
+  }
+
+  on (event, listener) {
+    super.on(event, listener)
+
+    this.state.client.products = this.eventNames()
+
+    this.scheduler.start()
+
+    return this
+  }
+
+  off (event, listener) {
+    super.off(event, listener)
+
+    this.state.client.products = this.eventNames()
+
+    if (!this.state.client.products.length) {
+      this.scheduler.stop()
+    }
+
+    return this
+  }
+
+  poll (cb) {
+    request(JSON.stringify(this.state), this.requestOptions, (err, data, statusCode) => {
+      // 404 means RC is disabled, ignore it
+      if (statusCode === 404) return cb()
+
+      if (err) {
+        log.error(err)
+        return cb()
+      }
+
+      // if error was just sent, reset the state
+      if (this.state.client.state.has_error) {
+        this.state.client.state.has_error = false
+        this.state.client.state.error = ''
+      }
+
+      if (data && data !== '{}') { // '{}' means the tracer is up to date
+        try {
+          this.parseConfig(JSON.parse(data))
+        } catch (err) {
+          log.error(`Could not parse remote config response: ${err}`)
+
+          this.state.client.state.has_error = true
+          this.state.client.state.error = err.toString()
+        }
+      }
+
+      cb()
+    })
+  }
+
+  // `client_configs` is the list of config paths to have applied
+  // `targets` is the signed index with metadata for config files
+  // `target_files` is the list of config files containing the actual config data
+  parseConfig ({
+    client_configs: clientConfigs = [],
+    targets,
+    target_files: targetFiles = []
+  }) {
+    const toUnapply = []
+    const toApply = []
+    const toModify = []
+
+    for (const appliedConfig of this.appliedConfigs.values()) {
+      if (!clientConfigs.includes(appliedConfig.path)) {
+        toUnapply.push(appliedConfig)
+      }
+    }
+
+    targets = fromBase64JSON(targets)
+
+    if (targets) {
+      for (const path of clientConfigs) {
+        const meta = targets.signed.targets[path]
+        if (!meta) throw new Error(`Unable to find target for path ${path}`)
+
+        const current = this.appliedConfigs.get(path)
+
+        const newConf = {}
+
+        if (current) {
+          if (current.hashes.sha256 === meta.hashes.sha256) continue
+
+          toModify.push(newConf)
+        } else {
+          toApply.push(newConf)
+        }
+
+        const file = targetFiles.find(file => file.path === path)
+        if (!file) throw new Error(`Unable to find file for path ${path}`)
+
+        // TODO: verify signatures
+        //       verify length
+        //       verify hash
+        //       verify _type
+        // TODO: new Date(meta.signed.expires) ignore the Targets data if it has expired ?
+
+        const { product, id } = parseConfigPath(path)
+
+        Object.assign(newConf, {
+          path,
+          product,
+          id,
+          version: meta.custom.v,
+          apply_state: 1,
+          apply_error: '',
+          length: meta.length,
+          hashes: meta.hashes,
+          file: fromBase64JSON(file.raw)
+        })
+      }
+
+      this.state.client.state.targets_version = targets.signed.version
+      this.state.client.state.backend_client_state = targets.signed.custom.opaque_backend_state
+    }
+
+    if (toUnapply.length || toApply.length || toModify.length) {
+      this.dispatch(toUnapply, 'unapply')
+      this.dispatch(toApply, 'apply')
+      this.dispatch(toModify, 'modify')
+
+      this.state.client.state.config_states = []
+      this.state.cached_target_files = []
+
+      for (const conf of this.appliedConfigs.values()) {
+        this.state.client.state.config_states.push({
+          id: conf.id,
+          version: conf.version,
+          product: conf.product,
+          apply_state: conf.apply_state,
+          apply_error: conf.apply_error
+        })
+
+        this.state.cached_target_files.push({
+          path: conf.path,
+          length: conf.length,
+          hashes: Object.entries(conf.hashes).map((entry) => ({ algorithm: entry[0], hash: entry[1] }))
+        })
+      }
+    }
+  }
+
+  dispatch (list, action) {
+    for (const item of list) {
+      try {
+        // TODO: do we want to pass old and new config ?
+        this.emit(item.product, action, item.file)
+
+        item.apply_state = 2
+      } catch (err) {
+        item.apply_state = 3
+        item.apply_error = err.toString()
+      }
+
+      if (action === 'unapply') {
+        this.appliedConfigs.delete(item.path)
+      } else {
+        this.appliedConfigs.set(item.path, item)
+      }
+    }
+  }
+}
+
+function fromBase64JSON (str) {
+  if (!str) return null
+
+  return JSON.parse(Buffer.from(str, 'base64').toString())
+}
+
+const configPathRegex = /^(?:datadog\/\d+|employee)\/([^/]+)\/([^/]+)\/[^/]+$/
+
+function parseConfigPath (configPath) {
+  const match = configPathRegex.exec(configPath)
+
+  if (!match || !match[1] || !match[2]) {
+    throw new Error(`Unable to parse path ${configPath}`)
+  }
+
+  return {
+    product: match[1],
+    id: match[2]
+  }
+}
+
+module.exports = RemoteConfigManager

package/packages/dd-trace/src/{exporters → appsec/remote_config}/scheduler.js

@@ -8,21 +8,21 @@ class Scheduler {
   }
 
   start () {
-    this._timer = setInterval(this._callback, this._interval)
-    this._timer.unref && this._timer.unref()
+    if (this._timer) return
 
-    process.once('beforeExit', this._callback)
+    this.runAfterDelay(0)
   }
 
-  stop () {
-    clearInterval(this._timer)
+  runAfterDelay (interval = this._interval) {
+    this._timer = setTimeout(this._callback, interval, () => this.runAfterDelay())
 
-    process.removeListener('beforeExit', this._callback)
+    this._timer.unref && this._timer.unref()
   }
 
-  reset () {
-    this.stop()
-    this.start()
+  stop () {
+    clearTimeout(this._timer)
+
+    this._timer = null
   }
 }
 

package/packages/dd-trace/src/appsec/rule_manager.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const callbacks = require('./callbacks')
+const Gateway = require('./gateway/engine')
 
 const appliedCallbacks = new Map()
 
@@ -14,6 +15,8 @@ function applyRules (rules, config) {
 }
 
 function clearAllRules () {
+  Gateway.manager.clear()
+
   for (const [key, callback] of appliedCallbacks) {
     callback.clear()
 

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -35,9 +35,7 @@ function getCommonRequestOptions (url) {
       'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY
     },
     timeout: 15000,
-    protocol: url.protocol,
-    hostname: url.hostname,
-    port: url.port
+    url
   }
 }
 
@@ -71,16 +69,16 @@ function getCommitsToExclude ({ url, repositoryUrl }, callback) {
     }))
   })
 
-  request(localCommitData, options, (err, response, statusCode) => {
+  request(localCommitData, options, (err, response) => {
     if (err) {
-      const error = new Error(`search_commits returned an error: status code ${statusCode}`)
+      const error = new Error(`Error fetching commits to exclude: ${err.message}`)
       return callback(error)
     }
     let commitsToExclude
     try {
       commitsToExclude = sanitizeCommits(JSON.parse(response).data)
     } catch (e) {
-      return callback(new Error(`Can't parse search_commits response: ${e.message}`))
+      return callback(new Error(`Can't parse commits to exclude response: ${e.message}`))
     }
     callback(null, commitsToExclude, headCommit)
   })
@@ -129,7 +127,7 @@ function uploadPackFile ({ url, packFileToUpload, repositoryUrl, headCommit }, c
   }
   request(form, options, (err, _, statusCode) => {
     if (err) {
-      const error = new Error(`Could not upload packfiles: status code ${statusCode}`)
+      const error = new Error(`Could not upload packfiles: status code ${statusCode}: ${err.message}`)
       return callback(error)
     }
     callback(null)

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js

@@ -36,9 +36,7 @@ function getItrConfiguration ({
       'dd-application-key': appKey,
       'Content-Type': 'application/json'
     },
-    protocol: intakeUrl.protocol,
-    hostname: intakeUrl.hostname,
-    port: intakeUrl.port
+    url: intakeUrl
   }
 
   const data = JSON.stringify({

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js

@@ -34,9 +34,7 @@ function getSkippableSuites ({
       'Content-Type': 'application/json'
     },
     timeout: 15000,
-    protocol: intakeUrl.protocol,
-    hostname: intakeUrl.hostname,
-    port: intakeUrl.port
+    url: intakeUrl
   }
 
   const data = JSON.stringify({

package/packages/dd-trace/src/config.js

@@ -95,6 +95,11 @@ class Config {
       process.env.DD_RUNTIME_METRICS_ENABLED,
       false
     )
+    const DD_DBM_PROPAGATION_MODE = coalesce(
+      options.dbmPropagationMode,
+      process.env.DD_DBM_PROPAGATION_MODE,
+      'disabled'
+    )
     const DD_AGENT_HOST = coalesce(
       options.hostname,
       process.env.DD_AGENT_HOST,
@@ -113,6 +118,7 @@ class Config {
       null
     )
     const DD_CIVISIBILITY_AGENTLESS_URL = process.env.DD_CIVISIBILITY_AGENTLESS_URL
+    const DD_CIVISIBILITY_AGENTLESS_ENABLED = process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED
 
     const DD_CIVISIBILITY_ITR_ENABLED = coalesce(
       process.env.DD_CIVISIBILITY_ITR_ENABLED,
@@ -201,16 +207,21 @@ class Config {
       false
     )
 
-    let appsec = options.appsec || (options.experimental && options.experimental.appsec)
+    let appsec = options.appsec != null ? options.appsec : options.experimental && options.experimental.appsec
+
+    if (typeof appsec === 'boolean') {
+      appsec = {
+        enabled: appsec
+      }
+    } else if (appsec == null) {
+      appsec = {}
+    }
 
     const DD_APPSEC_ENABLED = coalesce(
-      appsec && (appsec === true || appsec.enabled === true), // TODO: remove when enabled by default
-      process.env.DD_APPSEC_ENABLED,
-      false
+      appsec.enabled,
+      process.env.DD_APPSEC_ENABLED && isTrue(process.env.DD_APPSEC_ENABLED)
     )
 
-    appsec = appsec || {}
-
     const DD_APPSEC_RULES = coalesce(
       appsec.rules,
       process.env.DD_APPSEC_RULES,
@@ -310,6 +321,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     const defaultFlushInterval = inAWSLambda ? 0 : 2000
 
     this.tracing = !isFalse(DD_TRACING_ENABLED)
+    this.dbmPropagationMode = DD_DBM_PROPAGATION_MODE
     this.logInjection = isTrue(DD_LOGS_INJECTION)
     this.env = DD_ENV
     this.url = DD_CIVISIBILITY_AGENTLESS_URL ? new URL(DD_CIVISIBILITY_AGENTLESS_URL)
@@ -356,7 +368,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION
     this.tagsHeaderMaxLength = parseInt(DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH)
     this.appsec = {
-      enabled: isTrue(DD_APPSEC_ENABLED),
+      enabled: DD_APPSEC_ENABLED,
       rules: DD_APPSEC_RULES,
       rateLimit: DD_APPSEC_TRACE_RATE_LIMIT,
       wafTimeout: DD_APPSEC_WAF_TIMEOUT,
@@ -369,8 +381,12 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       maxConcurrentRequests: DD_IAST_MAX_CONCURRENT_REQUESTS,
       maxContextOperations: DD_IAST_MAX_CONTEXT_OPERATIONS
     }
-    this.isGitUploadEnabled = isTrue(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED)
-    this.isIntelligentTestRunnerEnabled = isTrue(DD_CIVISIBILITY_ITR_ENABLED)
+
+    const isCiVisibilityAgentlessEnabled = isTrue(DD_CIVISIBILITY_AGENTLESS_ENABLED)
+    this.isIntelligentTestRunnerEnabled = isCiVisibilityAgentlessEnabled && isTrue(DD_CIVISIBILITY_ITR_ENABLED)
+    this.isGitUploadEnabled = this.isIntelligentTestRunnerEnabled ||
+      (isCiVisibilityAgentlessEnabled && isTrue(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED))
+
     this.stats = {
       enabled: isTrue(DD_TRACE_STATS_COMPUTATION_ENABLED)
     }

package/packages/dd-trace/src/constants.js

@@ -19,5 +19,10 @@ module.exports = {
   SPAN_SAMPLING_RULE_RATE: '_dd.span_sampling.rule_rate',
   SPAN_SAMPLING_MAX_PER_SECOND: '_dd.span_sampling.max_per_second',
   DATADOG_LAMBDA_EXTENSION_PATH: '/opt/extensions/datadog-agent',
-  DECISION_MAKER_KEY: '_dd.p.dm'
+  DECISION_MAKER_KEY: '_dd.p.dm',
+  PROCESS_ID: 'process_id',
+  ERROR_TYPE: 'error.type',
+  ERROR_MESSAGE: 'error.message',
+  ERROR_STACK: 'error.stack',
+  COMPONENT: 'component'
 }

package/packages/dd-trace/src/exporters/common/request.js

@@ -66,7 +66,13 @@ function request (data, options, callback) {
       if (res.statusCode >= 200 && res.statusCode <= 299) {
         callback(null, responseData, res.statusCode)
       } else {
-        const error = new Error(`Error from the endpoint: ${res.statusCode} ${http.STATUS_CODES[res.statusCode]}`)
+        const fullUrl = `${options.url || options.hostname || `localhost:${options.port}`}${options.path}`
+        // eslint-disable-next-line
+        let errorMessage = `Error from ${fullUrl}: ${res.statusCode} ${http.STATUS_CODES[res.statusCode]}.`
+        if (responseData) {
+          errorMessage += ` Response from the endpoint: "${responseData}"`
+        }
+        const error = new Error(errorMessage)
         error.status = res.statusCode
 
         callback(error, null, res.statusCode)

package/packages/dd-trace/src/format.js

@@ -17,6 +17,10 @@ const MEASURED = tags.MEASURED
 const ORIGIN_KEY = constants.ORIGIN_KEY
 const HOSTNAME_KEY = constants.HOSTNAME_KEY
 const TOP_LEVEL_KEY = constants.TOP_LEVEL_KEY
+const PROCESS_ID = constants.PROCESS_ID
+const ERROR_MESSAGE = constants.ERROR_MESSAGE
+const ERROR_STACK = constants.ERROR_STACK
+const ERROR_TYPE = constants.ERROR_TYPE
 
 const map = {
   'service.name': 'service',
@@ -89,9 +93,9 @@ function extractTags (trace, span) {
           extractError(trace, tags[tag])
         }
         break
-      case 'error.type':
-      case 'error.msg':
-      case 'error.stack':
+      case ERROR_TYPE:
+      case ERROR_MESSAGE:
+      case ERROR_STACK:
         // HACK: remove when implemented in the backend
         if (context._name !== 'fs.operation') {
           trace.error = 1
@@ -103,12 +107,10 @@ function extractTags (trace, span) {
     }
   }
 
-  if (span.tracer()._service === tags['service.name']) {
-    addTag(trace.meta, trace.metrics, 'language', 'javascript')
-  }
-
   setSingleSpanIngestionTags(trace, context._sampling.spanSampling)
 
+  addTag(trace.meta, trace.metrics, 'language', 'javascript')
+  addTag(trace.meta, trace.metrics, PROCESS_ID, process.pid)
   addTag(trace.meta, trace.metrics, SAMPLING_PRIORITY_KEY, priority)
   addTag(trace.meta, trace.metrics, ORIGIN_KEY, origin)
   addTag(trace.meta, trace.metrics, HOSTNAME_KEY, hostname)
@@ -144,9 +146,9 @@ function extractError (trace, error) {
   trace.error = 1
 
   if (isError(error)) {
-    addTag(trace.meta, trace.metrics, 'error.msg', error.message)
-    addTag(trace.meta, trace.metrics, 'error.type', error.name)
-    addTag(trace.meta, trace.metrics, 'error.stack', error.stack)
+    addTag(trace.meta, trace.metrics, ERROR_MESSAGE, error.message)
+    addTag(trace.meta, trace.metrics, ERROR_TYPE, error.name)
+    addTag(trace.meta, trace.metrics, ERROR_STACK, error.stack)
   }
 }
 

package/packages/dd-trace/src/opentracing/propagation/text_map.js

@@ -130,11 +130,7 @@ class TextMapPropagator {
 
   _injectTraceparent (spanContext, carrier) {
     if (!this._config.experimental.traceparent) return
-
-    const sampling = spanContext._sampling.priority >= AUTO_KEEP ? '01' : '00'
-    const traceId = spanContext._traceId.toString(16).padStart(32, '0')
-    const spanId = spanContext._spanId.toString(16).padStart(16, '0')
-    carrier[traceparentKey] = `01-${traceId}-${spanId}-${sampling}`
+    carrier[traceparentKey] = spanContext.toTraceparent()
   }
 
   _extractSpanContext (carrier) {

package/packages/dd-trace/src/opentracing/span_context.js

@@ -1,5 +1,7 @@
 'use strict'
 
+const { AUTO_KEEP } = require('../../../../ext/priority')
+
 class DatadogSpanContext {
   constructor (props) {
     props = props || {}
@@ -27,6 +29,13 @@ class DatadogSpanContext {
   toSpanId () {
     return this._spanId.toString(10)
   }
+
+  toTraceparent () {
+    const sampling = this._sampling.priority >= AUTO_KEEP ? '01' : '00'
+    const traceId = this._traceId.toString(16).padStart(32, '0')
+    const spanId = this._spanId.toString(16).padStart(16, '0')
+    return `01-${traceId}-${spanId}-${sampling}`
+  }
 }
 
 module.exports = DatadogSpanContext

package/packages/dd-trace/src/plugin_manager.js

@@ -125,7 +125,8 @@ module.exports = class PluginManager {
       isIntelligentTestRunnerEnabled,
       site,
       experimental,
-      url
+      url,
+      dbmPropagationMode
     } = this._tracerConfig
 
     const sharedConfig = {}
@@ -152,6 +153,8 @@ module.exports = class PluginManager {
 
     sharedConfig.isIntelligentTestRunnerEnabled = isIntelligentTestRunnerEnabled
 
+    sharedConfig.dbmPropagationMode = dbmPropagationMode
+
     if (serviceMapping && serviceMapping[name]) {
       sharedConfig.service = serviceMapping[name]
     }