dd-trace 2.39.0 → 2.41.0

package/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js

@@ -4,30 +4,45 @@ const TaintedUtils = require('@datadog/native-iast-taint-tracking')
 const { storage } = require('../../../../../datadog-core')
 const iastContextFunctions = require('../iast-context')
 const iastLog = require('../iast-log')
+const { EXECUTED_PROPAGATION } = require('../telemetry/iast-metric')
+const { isDebugAllowed } = require('../telemetry/verbosity')
 
 function noop (res) { return res }
-const TaintTrackingDummy = {
+// NOTE: methods of this object must be synchronized with csi-methods.js file definitions!
+// Otherwise you may end up rewriting a method and not providing its rewritten implementation
+const TaintTrackingNoop = {
   plusOperator: noop,
-  trim: noop,
-  trimEnd: noop,
   concat: noop,
-  substring: noop,
-  substr: noop,
+  replace: noop,
   slice: noop,
-  replace: noop
+  substr: noop,
+  substring: noop,
+  trim: noop,
+  trimEnd: noop
 }
 
-function getTransactionId () {
-  const store = storage.getStore()
-  const iastContext = iastContextFunctions.getIastContext(store)
+function getTransactionId (iastContext) {
   return iastContext && iastContext[iastContextFunctions.IAST_TRANSACTION_ID]
 }
 
-function getFilteredCsiFn (cb, filter) {
+function getContextDefault () {
+  const store = storage.getStore()
+  return iastContextFunctions.getIastContext(store)
+}
+
+function getContextDebug () {
+  const iastContext = getContextDefault()
+  EXECUTED_PROPAGATION.inc(null, iastContext)
+  return iastContext
+}
+
+function getFilteredCsiFn (cb, filter, getContext) {
   return function csiCall (res, fn, target, ...rest) {
     try {
       if (filter(res, fn, target)) { return res }
-      const transactionId = getTransactionId()
+
+      const context = getContext()
+      const transactionId = getTransactionId(context)
       if (transactionId) {
         return cb(transactionId, res, target, ...rest)
       }
@@ -47,7 +62,7 @@ function isValidCsiMethod (fn, protos) {
   return protos.some(proto => fn === proto)
 }
 
-function getCsiFn (cb, ...protos) {
+function getCsiFn (cb, getContext, ...protos) {
   let filter
   if (!protos || protos.length === 0) {
     filter = (res, fn, target) => notString(res, target)
@@ -57,49 +72,73 @@ function getCsiFn (cb, ...protos) {
   } else {
     filter = (res, fn, target) => notString(res, target) || !isValidCsiMethod(fn, protos)
   }
-  return getFilteredCsiFn(cb, filter)
+  return getFilteredCsiFn(cb, filter, getContext)
 }
 
-function csiMethodsDefaults (names, excluded) {
+function csiMethodsDefaults (names, excluded, getContext) {
   const impl = {}
   names.forEach(name => {
     if (excluded.indexOf(name) !== -1) return
     impl[name] = getCsiFn(
       (transactionId, res, target, ...rest) => TaintedUtils[name](transactionId, res, target, ...rest),
+      getContext,
       String.prototype[name]
     )
   })
   return impl
 }
 
-const csiMethodsOverrides = {
-  plusOperator: function (res, op1, op2) {
-    try {
-      if (notString(res) || (notString(op1) && notString(op2))) { return res }
-      const transactionId = getTransactionId()
-      if (transactionId) {
-        return TaintedUtils.concat(transactionId, res, op1, op2)
+function csiMethodsOverrides (getContext) {
+  return {
+    plusOperator: function (res, op1, op2) {
+      try {
+        if (notString(res) || (notString(op1) && notString(op2))) { return res }
+        const iastContext = getContext()
+        const transactionId = getTransactionId(iastContext)
+        if (transactionId) {
+          return TaintedUtils.concat(transactionId, res, op1, op2)
+        }
+      } catch (e) {
+        iastLog.error(`Error invoking CSI plusOperator`)
+          .errorAndPublish(e)
       }
-    } catch (e) {
-      iastLog.error(`Error invoking CSI plusOperator`)
-        .errorAndPublish(e)
-    }
-    return res
-  },
+      return res
+    },
+
+    trim: getCsiFn(
+      (transactionId, res, target) => TaintedUtils.trim(transactionId, res, target),
+      getContext,
+      String.prototype.trim,
+      String.prototype.trimStart
+    )
+  }
+}
+
+function createImplWith (getContext) {
+  const methodNames = Object.keys(TaintTrackingNoop)
+  const overrides = csiMethodsOverrides(getContext)
+
+  // impls could be cached but at the moment there is only one invocation to getTaintTrackingImpl
+  return {
+    ...csiMethodsDefaults(methodNames, Object.keys(overrides), getContext),
+    ...overrides
+  }
+}
+
+function getTaintTrackingImpl (telemetryVerbosity, dummy = false) {
+  if (dummy) return TaintTrackingNoop
 
-  trim: getCsiFn(
-    (transactionId, res, target) => TaintedUtils.trim(transactionId, res, target),
-    String.prototype.trim,
-    String.prototype.trimStart
-  )
+  // with Verbosity.DEBUG every invocation of a TaintedUtils method increases the EXECUTED_PROPAGATION metric
+  return isDebugAllowed(telemetryVerbosity)
+    ? createImplWith(getContextDebug)
+    : createImplWith(getContextDefault)
 }
 
-const TaintTracking = {
-  ...csiMethodsDefaults(Object.keys(TaintTrackingDummy), Object.keys(csiMethodsOverrides)),
-  ...csiMethodsOverrides
+function getTaintTrackingNoop () {
+  return getTaintTrackingImpl(null, true)
 }
 
 module.exports = {
-  TaintTracking,
-  TaintTrackingDummy
+  getTaintTrackingImpl,
+  getTaintTrackingNoop
 }

package/packages/dd-trace/src/appsec/iast/telemetry/iast-metric.js

@@ -0,0 +1,101 @@
+'use strict'
+
+const { getNamespaceFromContext, globalNamespace } = require('./namespaces')
+
+const Scope = {
+  GLOBAL: 'GLOBAL',
+  REQUEST: 'REQUEST'
+}
+
+const PropagationType = {
+  STRING: 'STRING',
+  JSON: 'JSON',
+  URL: 'URL'
+}
+
+const TagKey = {
+  VULNERABILITY_TYPE: 'vulnerability_type',
+  SOURCE_TYPE: 'source_type',
+  PROPAGATION_TYPE: 'propagation_type'
+}
+
+class IastMetric {
+  constructor (name, scope, tagKey) {
+    this.name = name
+    this.scope = scope
+    this.tagKey = tagKey
+  }
+
+  getNamespace (context) {
+    return getNamespaceFromContext(context) || globalNamespace
+  }
+
+  getTag (tagValue) {
+    return tagValue ? { [this.tagKey]: tagValue } : undefined
+  }
+
+  addValue (value, tagValue, context) {
+    this.getNamespace(context)
+      .count(this.name, this.getTag(tagValue))
+      .inc(value)
+  }
+
+  add (value, tagValue, context) {
+    if (Array.isArray(tagValue)) {
+      tagValue.forEach(tag => this.addValue(value, tag, context))
+    } else {
+      this.addValue(value, tagValue, context)
+    }
+  }
+
+  inc (tagValue, context) {
+    this.add(1, tagValue, context)
+  }
+}
+
+function getExecutedMetric (tagKey) {
+  return tagKey === TagKey.VULNERABILITY_TYPE ? EXECUTED_SINK : EXECUTED_SOURCE
+}
+
+function getInstrumentedMetric (tagKey) {
+  return tagKey === TagKey.VULNERABILITY_TYPE ? INSTRUMENTED_SINK : INSTRUMENTED_SOURCE
+}
+
+const INSTRUMENTED_PROPAGATION = new IastMetric('instrumented.propagation', Scope.GLOBAL)
+const INSTRUMENTED_SOURCE = new IastMetric('instrumented.source', Scope.GLOBAL, TagKey.SOURCE_TYPE)
+const INSTRUMENTED_SINK = new IastMetric('instrumented.sink', Scope.GLOBAL, TagKey.VULNERABILITY_TYPE)
+
+const EXECUTED_SOURCE = new IastMetric('executed.source', Scope.REQUEST, TagKey.SOURCE_TYPE)
+const EXECUTED_SINK = new IastMetric('executed.sink', Scope.REQUEST, TagKey.VULNERABILITY_TYPE)
+
+const REQUEST_TAINTED = new IastMetric('request.tainted', Scope.REQUEST)
+
+// DEBUG using metrics
+const EXECUTED_PROPAGATION = new IastMetric('executed.propagation', Scope.REQUEST)
+const EXECUTED_TAINTED = new IastMetric('executed.tainted', Scope.REQUEST)
+
+// DEBUG using distribution endpoint
+const INSTRUMENTATION_TIME = new IastMetric('instrumentation.time', Scope.GLOBAL)
+
+module.exports = {
+  INSTRUMENTED_PROPAGATION,
+  INSTRUMENTED_SOURCE,
+  INSTRUMENTED_SINK,
+
+  EXECUTED_PROPAGATION,
+  EXECUTED_SOURCE,
+  EXECUTED_SINK,
+  EXECUTED_TAINTED,
+
+  REQUEST_TAINTED,
+
+  INSTRUMENTATION_TIME,
+
+  PropagationType,
+  TagKey,
+
+  IastMetric,
+
+  getExecutedMetric,
+  getInstrumentedMetric
+}

package/packages/dd-trace/src/appsec/iast/telemetry/index.js

@@ -0,0 +1,45 @@
+'use strict'
+
+const telemetryMetrics = require('../../../telemetry/metrics')
+const telemetryLogs = require('./log')
+const { Verbosity, getVerbosity } = require('./verbosity')
+const { initRequestNamespace, finalizeRequestNamespace, globalNamespace } = require('./namespaces')
+
+class Telemetry {
+  configure (config, verbosity) {
+    // in order to telemetry be enabled, tracer telemetry and metrics collection have to be enabled
+    this.enabled = config && config.telemetry && config.telemetry.enabled && config.telemetry.metrics
+    this.verbosity = this.enabled ? getVerbosity(verbosity) : Verbosity.OFF
+
+    if (this.enabled) {
+      telemetryMetrics.manager.set('iast', globalNamespace)
+    }
+
+    telemetryLogs.start()
+  }
+
+  stop () {
+    this.enabled = false
+    telemetryMetrics.manager.delete('iast')
+
+    telemetryLogs.stop()
+  }
+
+  isEnabled () {
+    return this.enabled
+  }
+
+  onRequestStart (context) {
+    if (this.isEnabled() && this.verbosity !== Verbosity.OFF) {
+      initRequestNamespace(context)
+    }
+  }
+
+  onRequestEnd (context, rootSpan) {
+    if (this.isEnabled() && this.verbosity !== Verbosity.OFF) {
+      finalizeRequestNamespace(context, rootSpan)
+    }
+  }
+}
+
+module.exports = new Telemetry()

package/packages/dd-trace/src/appsec/iast/telemetry/{logs.js → log/index.js}

@@ -1,9 +1,9 @@
 'use strict'
 
-const dc = require('../../../../../diagnostics_channel')
-const logCollector = require('./log_collector')
-const { sendData } = require('../../../telemetry/send-data')
-const log = require('../../../log')
+const dc = require('../../../../../../diagnostics_channel')
+const logCollector = require('./log-collector')
+const { sendData } = require('../../../../telemetry/send-data')
+const log = require('../../../../log')
 
 const telemetryStartChannel = dc.channel('datadog:telemetry:start')
 const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
@@ -37,7 +37,7 @@ function isLogCollectionEnabled (config) {
 
 function onTelemetryStart (msg) {
   if (!msg || !isLogCollectionEnabled(msg.config)) {
-    log.info('IAST telemetry logs start received but log collection is not enabled or configuration is incorrect')
+    log.info('IAST telemetry logs start event received but log collection is not enabled or configuration is incorrect')
     return false
   }
 

package/packages/dd-trace/src/appsec/iast/telemetry/{log_collector.js → log/log-collector.js}

@@ -1,6 +1,6 @@
 'use strict'
 
-const log = require('../../../log')
+const log = require('../../../../log')
 
 const logs = new Map()
 

package/packages/dd-trace/src/appsec/iast/telemetry/namespaces.js

@@ -0,0 +1,76 @@
+'use strict'
+
+const log = require('../../../log')
+const { Namespace } = require('../../../telemetry/metrics')
+const { addMetricsToSpan, filterTags } = require('./span-tags')
+const { IAST_TRACE_METRIC_PREFIX } = require('../tags')
+
+const DD_IAST_METRICS_NAMESPACE = Symbol('_dd.iast.request.metrics.namespace')
+
+function initRequestNamespace (context) {
+  if (!context) return
+
+  const namespace = new Namespace('iast')
+  context[DD_IAST_METRICS_NAMESPACE] = namespace
+  return namespace
+}
+
+function getNamespaceFromContext (context) {
+  return context && context[DD_IAST_METRICS_NAMESPACE]
+}
+
+function finalizeRequestNamespace (context, rootSpan) {
+  try {
+    const namespace = getNamespaceFromContext(context)
+    if (!namespace) return
+
+    const metrics = [...namespace.metrics.values()]
+    namespace.metrics.clear()
+
+    addMetricsToSpan(rootSpan, metrics, IAST_TRACE_METRIC_PREFIX)
+
+    merge(metrics)
+  } catch (e) {
+    log.error(e)
+  } finally {
+    if (context) {
+      delete context[DD_IAST_METRICS_NAMESPACE]
+    }
+  }
+}
+
+function merge (metrics) {
+  metrics.forEach(metric => metric.points.forEach(point => {
+    globalNamespace
+      .count(metric.metric, getTagsObject(metric.tags))
+      .inc(point[1])
+  }))
+}
+
+function getTagsObject (tags) {
+  if (tags && tags.length > 0) {
+    return filterTags(tags)
+  }
+}
+
+class IastNamespace extends Namespace {
+  constructor () {
+    super('iast')
+  }
+
+  reset () {
+    this.metrics.clear()
+    this.distributions.clear()
+  }
+}
+
+const globalNamespace = new IastNamespace()
+
+module.exports = {
+  initRequestNamespace,
+  getNamespaceFromContext,
+  finalizeRequestNamespace,
+  globalNamespace,
+
+  DD_IAST_METRICS_NAMESPACE
+}

package/packages/dd-trace/src/appsec/iast/telemetry/span-tags.js

@@ -0,0 +1,53 @@
+'use strict'
+
+function addMetricsToSpan (rootSpan, metrics, tagPrefix) {
+  if (!rootSpan || !rootSpan.addTags || !metrics) return
+
+  const flattenMap = new Map()
+  metrics
+    .filter(data => data && data.metric)
+    .forEach(data => {
+      const name = taggedMetricName(data)
+      let total = flattenMap.get(name)
+      const value = flatten(data)
+      if (!total) {
+        total = value
+      } else {
+        total += value
+      }
+      flattenMap.set(name, total)
+    })
+
+  for (const [key, value] of flattenMap) {
+    const tagName = `${tagPrefix}.${key}`
+    rootSpan.addTags({
+      [tagName]: value
+    })
+  }
+}
+
+function flatten (metricData) {
+  return metricData.points && metricData.points.map(point => point[1]).reduce((total, value) => total + value, 0)
+}
+
+function taggedMetricName (data) {
+  const metric = data.metric
+  const tags = data.tags && filterTags(data.tags)
+  return !tags || !tags.length
+    ? metric
+    : `${metric}.${processTagValue(tags)}`
+}
+
+function filterTags (tags) {
+  return tags.filter(tag => !tag.startsWith('lib_language') && !tag.startsWith('version'))
+}
+
+function processTagValue (tags) {
+  return tags.map(tag => tag.includes(':') ? tag.split(':')[1] : tag)
+    .join('_').replace(/\./g, '_')
+}
+
+module.exports = {
+  addMetricsToSpan,
+  filterTags
+}

package/packages/dd-trace/src/appsec/iast/telemetry/verbosity.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const Verbosity = {
+  OFF: 0,
+  MANDATORY: 1,
+  INFORMATION: 2,
+  DEBUG: 3
+}
+
+function isDebugAllowed (value) {
+  return value >= Verbosity.DEBUG
+}
+
+function isInfoAllowed (value) {
+  return value >= Verbosity.INFORMATION
+}
+
+function getVerbosity (verbosity) {
+  if (verbosity) {
+    verbosity = verbosity.toUpperCase()
+    return Verbosity[verbosity] !== undefined ? Verbosity[verbosity] : Verbosity.INFORMATION
+  } else {
+    return Verbosity.INFORMATION
+  }
+}
+
+function getName (verbosityValue) {
+  for (const name in Verbosity) {
+    if (Verbosity[name] === verbosityValue) {
+      return name
+    }
+  }
+  return 'OFF'
+}
+
+module.exports = {
+  Verbosity,
+  isDebugAllowed,
+  isInfoAllowed,
+  getVerbosity,
+  getName
+}

package/packages/dd-trace/src/config.js

@@ -244,6 +244,10 @@ class Config {
       process.env.DD_TELEMETRY_DEBUG,
       false
     )
+    const DD_TELEMETRY_METRICS_ENABLED = coalesce(
+      process.env.DD_TELEMETRY_METRICS_ENABLED,
+      false
+    )
     const DD_TRACE_AGENT_PROTOCOL_VERSION = coalesce(
       options.protocolVersion,
       process.env.DD_TRACE_AGENT_PROTOCOL_VERSION,
@@ -316,6 +320,12 @@ class Config {
     const DD_TRACE_SPAN_ATTRIBUTE_SCHEMA = validateNamingVersion(
       process.env.DD_TRACE_SPAN_ATTRIBUTE_SCHEMA
     )
+    const DD_TRACE_PEER_SERVICE_MAPPING = coalesce(
+      options.peerServiceMapping,
+      process.env.DD_TRACE_PEER_SERVICE_MAPPING ? fromEntries(
+        process.env.DD_TRACE_PEER_SERVICE_MAPPING.split(',').map(x => x.trim().split(':'))
+      ) : {}
+    )
     const DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED = process.env.DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED
 
     const DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED = coalesce(
@@ -458,6 +468,12 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       true
     )
 
+    const DD_IAST_TELEMETRY_VERBOSITY = coalesce(
+      iastOptions && iastOptions.telemetryVerbosity,
+      process.env.DD_IAST_TELEMETRY_VERBOSITY,
+      'INFORMATION'
+    )
+
     const DD_CIVISIBILITY_GIT_UPLOAD_ENABLED = coalesce(
       process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED,
       true
@@ -551,6 +567,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       : true
     )
     this.traceRemoveIntegrationServiceNamesEnabled = DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED
+    this.peerServiceMapping = DD_TRACE_PEER_SERVICE_MAPPING
     this.lookup = options.lookup
     this.startupLogs = isTrue(DD_TRACE_STARTUP_LOGS)
     // Disabled for CI Visibility's agentless
@@ -558,7 +575,8 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       enabled: DD_TRACE_EXPORTER !== 'datadog' && isTrue(DD_TRACE_TELEMETRY_ENABLED),
       heartbeatInterval: DD_TELEMETRY_HEARTBEAT_INTERVAL,
       logCollection: isTrue(DD_TELEMETRY_LOG_COLLECTION_ENABLED),
-      debug: isTrue(DD_TELEMETRY_DEBUG)
+      debug: isTrue(DD_TELEMETRY_DEBUG),
+      metrics: isTrue(DD_TELEMETRY_METRICS_ENABLED)
     }
     this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION
     this.tagsHeaderMaxLength = parseInt(DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH)
@@ -587,7 +605,8 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       maxConcurrentRequests: DD_IAST_MAX_CONCURRENT_REQUESTS,
       maxContextOperations: DD_IAST_MAX_CONTEXT_OPERATIONS,
       deduplicationEnabled: DD_IAST_DEDUPLICATION_ENABLED,
-      redactionEnabled: DD_IAST_REDACTION_ENABLED
+      redactionEnabled: DD_IAST_REDACTION_ENABLED,
+      telemetryVerbosity: DD_IAST_TELEMETRY_VERBOSITY
     }
 
     this.isCiVisibility = isTrue(DD_IS_CIVISIBILITY)

package/packages/dd-trace/src/constants.js

@@ -28,6 +28,7 @@ module.exports = {
   CLIENT_PORT_KEY: 'network.destination.port',
   PEER_SERVICE_KEY: 'peer.service',
   PEER_SERVICE_SOURCE_KEY: '_dd.peer.service.source',
+  PEER_SERVICE_REMAP_KEY: '_dd.peer.service.remapped_from',
   SCI_REPOSITORY_URL: '_dd.git.repository_url',
   SCI_COMMIT_SHA: '_dd.git.commit.sha'
 }

package/packages/dd-trace/src/dogstatsd.js

@@ -143,4 +143,17 @@ class DogStatsDClient {
   }
 }
 
-module.exports = DogStatsDClient
+class NoopDogStatsDClient {
+  gauge () { }
+
+  increment () { }
+
+  distribution () { }
+
+  flush () { }
+}
+
+module.exports = {
+  DogStatsDClient,
+  NoopDogStatsDClient
+}

package/packages/dd-trace/src/metrics.js

@@ -5,7 +5,7 @@
 const { URL, format } = require('url')
 const v8 = require('v8')
 const os = require('os')
-const Client = require('./dogstatsd')
+const { DogStatsDClient } = require('./dogstatsd')
 const log = require('./log')
 const Histogram = require('./histogram')
 const { performance } = require('perf_hooks')
@@ -67,7 +67,7 @@ module.exports = {
       }))
     }
 
-    client = new Client(clientConfig)
+    client = new DogStatsDClient(clientConfig)
 
     time = process.hrtime()
 

package/packages/dd-trace/src/opentracing/tracer.js

@@ -27,6 +27,7 @@ class DatadogTracer {
     this._env = config.env
     this._tags = config.tags
     this._computePeerService = config.spanComputePeerService
+    this._peerServiceMapping = config.peerServiceMapping
     this._logInjection = config.logInjection
     this._debug = config.debug
     this._prioritySampler = new PrioritySampler(config.env, config.sampler)

package/packages/dd-trace/src/plugins/ci_plugin.js

@@ -111,7 +111,12 @@ module.exports = class CiPlugin extends Plugin {
     const childOf = getTestParentSpan(this.tracer)
 
     let testTags = {
-      ...getTestCommonTags(testName, testSuite, this.frameworkVersion),
+      ...getTestCommonTags(
+        testName,
+        testSuite,
+        this.frameworkVersion,
+        this.constructor.id
+      ),
       [COMPONENT]: this.constructor.id,
       ...extraTags
     }