dd-trace 2.40.0 → 2.41.0

package/packages/dd-trace/src/appsec/iast/telemetry/{logs.js → log/index.js}

@@ -1,9 +1,9 @@
 'use strict'
 
-const dc = require('../../../../../diagnostics_channel')
-const logCollector = require('./log_collector')
-const { sendData } = require('../../../telemetry/send-data')
-const log = require('../../../log')
+const dc = require('../../../../../../diagnostics_channel')
+const logCollector = require('./log-collector')
+const { sendData } = require('../../../../telemetry/send-data')
+const log = require('../../../../log')
 
 const telemetryStartChannel = dc.channel('datadog:telemetry:start')
 const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
@@ -37,7 +37,7 @@ function isLogCollectionEnabled (config) {
 
 function onTelemetryStart (msg) {
   if (!msg || !isLogCollectionEnabled(msg.config)) {
-    log.info('IAST telemetry logs start received but log collection is not enabled or configuration is incorrect')
+    log.info('IAST telemetry logs start event received but log collection is not enabled or configuration is incorrect')
     return false
   }
 

package/packages/dd-trace/src/appsec/iast/telemetry/{log_collector.js → log/log-collector.js}

@@ -1,6 +1,6 @@
 'use strict'
 
-const log = require('../../../log')
+const log = require('../../../../log')
 
 const logs = new Map()
 

package/packages/dd-trace/src/appsec/iast/telemetry/namespaces.js

@@ -0,0 +1,76 @@
+'use strict'
+
+const log = require('../../../log')
+const { Namespace } = require('../../../telemetry/metrics')
+const { addMetricsToSpan, filterTags } = require('./span-tags')
+const { IAST_TRACE_METRIC_PREFIX } = require('../tags')
+
+const DD_IAST_METRICS_NAMESPACE = Symbol('_dd.iast.request.metrics.namespace')
+
+function initRequestNamespace (context) {
+  if (!context) return
+
+  const namespace = new Namespace('iast')
+  context[DD_IAST_METRICS_NAMESPACE] = namespace
+  return namespace
+}
+
+function getNamespaceFromContext (context) {
+  return context && context[DD_IAST_METRICS_NAMESPACE]
+}
+
+function finalizeRequestNamespace (context, rootSpan) {
+  try {
+    const namespace = getNamespaceFromContext(context)
+    if (!namespace) return
+
+    const metrics = [...namespace.metrics.values()]
+    namespace.metrics.clear()
+
+    addMetricsToSpan(rootSpan, metrics, IAST_TRACE_METRIC_PREFIX)
+
+    merge(metrics)
+  } catch (e) {
+    log.error(e)
+  } finally {
+    if (context) {
+      delete context[DD_IAST_METRICS_NAMESPACE]
+    }
+  }
+}
+
+function merge (metrics) {
+  metrics.forEach(metric => metric.points.forEach(point => {
+    globalNamespace
+      .count(metric.metric, getTagsObject(metric.tags))
+      .inc(point[1])
+  }))
+}
+
+function getTagsObject (tags) {
+  if (tags && tags.length > 0) {
+    return filterTags(tags)
+  }
+}
+
+class IastNamespace extends Namespace {
+  constructor () {
+    super('iast')
+  }
+
+  reset () {
+    this.metrics.clear()
+    this.distributions.clear()
+  }
+}
+
+const globalNamespace = new IastNamespace()
+
+module.exports = {
+  initRequestNamespace,
+  getNamespaceFromContext,
+  finalizeRequestNamespace,
+  globalNamespace,
+
+  DD_IAST_METRICS_NAMESPACE
+}

package/packages/dd-trace/src/appsec/iast/telemetry/span-tags.js

@@ -0,0 +1,53 @@
+'use strict'
+
+function addMetricsToSpan (rootSpan, metrics, tagPrefix) {
+  if (!rootSpan || !rootSpan.addTags || !metrics) return
+
+  const flattenMap = new Map()
+  metrics
+    .filter(data => data && data.metric)
+    .forEach(data => {
+      const name = taggedMetricName(data)
+      let total = flattenMap.get(name)
+      const value = flatten(data)
+      if (!total) {
+        total = value
+      } else {
+        total += value
+      }
+      flattenMap.set(name, total)
+    })
+
+  for (const [key, value] of flattenMap) {
+    const tagName = `${tagPrefix}.${key}`
+    rootSpan.addTags({
+      [tagName]: value
+    })
+  }
+}
+
+function flatten (metricData) {
+  return metricData.points && metricData.points.map(point => point[1]).reduce((total, value) => total + value, 0)
+}
+
+function taggedMetricName (data) {
+  const metric = data.metric
+  const tags = data.tags && filterTags(data.tags)
+  return !tags || !tags.length
+    ? metric
+    : `${metric}.${processTagValue(tags)}`
+}
+
+function filterTags (tags) {
+  return tags.filter(tag => !tag.startsWith('lib_language') && !tag.startsWith('version'))
+}
+
+function processTagValue (tags) {
+  return tags.map(tag => tag.includes(':') ? tag.split(':')[1] : tag)
+    .join('_').replace(/\./g, '_')
+}
+
+module.exports = {
+  addMetricsToSpan,
+  filterTags
+}

package/packages/dd-trace/src/appsec/iast/telemetry/verbosity.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const Verbosity = {
+  OFF: 0,
+  MANDATORY: 1,
+  INFORMATION: 2,
+  DEBUG: 3
+}
+
+function isDebugAllowed (value) {
+  return value >= Verbosity.DEBUG
+}
+
+function isInfoAllowed (value) {
+  return value >= Verbosity.INFORMATION
+}
+
+function getVerbosity (verbosity) {
+  if (verbosity) {
+    verbosity = verbosity.toUpperCase()
+    return Verbosity[verbosity] !== undefined ? Verbosity[verbosity] : Verbosity.INFORMATION
+  } else {
+    return Verbosity.INFORMATION
+  }
+}
+
+function getName (verbosityValue) {
+  for (const name in Verbosity) {
+    if (Verbosity[name] === verbosityValue) {
+      return name
+    }
+  }
+  return 'OFF'
+}
+
+module.exports = {
+  Verbosity,
+  isDebugAllowed,
+  isInfoAllowed,
+  getVerbosity,
+  getName
+}

package/packages/dd-trace/src/config.js

@@ -244,6 +244,10 @@ class Config {
       process.env.DD_TELEMETRY_DEBUG,
       false
     )
+    const DD_TELEMETRY_METRICS_ENABLED = coalesce(
+      process.env.DD_TELEMETRY_METRICS_ENABLED,
+      false
+    )
     const DD_TRACE_AGENT_PROTOCOL_VERSION = coalesce(
       options.protocolVersion,
       process.env.DD_TRACE_AGENT_PROTOCOL_VERSION,
@@ -316,6 +320,12 @@ class Config {
     const DD_TRACE_SPAN_ATTRIBUTE_SCHEMA = validateNamingVersion(
       process.env.DD_TRACE_SPAN_ATTRIBUTE_SCHEMA
     )
+    const DD_TRACE_PEER_SERVICE_MAPPING = coalesce(
+      options.peerServiceMapping,
+      process.env.DD_TRACE_PEER_SERVICE_MAPPING ? fromEntries(
+        process.env.DD_TRACE_PEER_SERVICE_MAPPING.split(',').map(x => x.trim().split(':'))
+      ) : {}
+    )
     const DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED = process.env.DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED
 
     const DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED = coalesce(
@@ -458,6 +468,12 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       true
     )
 
+    const DD_IAST_TELEMETRY_VERBOSITY = coalesce(
+      iastOptions && iastOptions.telemetryVerbosity,
+      process.env.DD_IAST_TELEMETRY_VERBOSITY,
+      'INFORMATION'
+    )
+
     const DD_CIVISIBILITY_GIT_UPLOAD_ENABLED = coalesce(
       process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED,
       true
@@ -551,6 +567,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       : true
     )
     this.traceRemoveIntegrationServiceNamesEnabled = DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED
+    this.peerServiceMapping = DD_TRACE_PEER_SERVICE_MAPPING
     this.lookup = options.lookup
     this.startupLogs = isTrue(DD_TRACE_STARTUP_LOGS)
     // Disabled for CI Visibility's agentless
@@ -558,7 +575,8 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       enabled: DD_TRACE_EXPORTER !== 'datadog' && isTrue(DD_TRACE_TELEMETRY_ENABLED),
       heartbeatInterval: DD_TELEMETRY_HEARTBEAT_INTERVAL,
       logCollection: isTrue(DD_TELEMETRY_LOG_COLLECTION_ENABLED),
-      debug: isTrue(DD_TELEMETRY_DEBUG)
+      debug: isTrue(DD_TELEMETRY_DEBUG),
+      metrics: isTrue(DD_TELEMETRY_METRICS_ENABLED)
     }
     this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION
     this.tagsHeaderMaxLength = parseInt(DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH)
@@ -587,7 +605,8 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       maxConcurrentRequests: DD_IAST_MAX_CONCURRENT_REQUESTS,
       maxContextOperations: DD_IAST_MAX_CONTEXT_OPERATIONS,
       deduplicationEnabled: DD_IAST_DEDUPLICATION_ENABLED,
-      redactionEnabled: DD_IAST_REDACTION_ENABLED
+      redactionEnabled: DD_IAST_REDACTION_ENABLED,
+      telemetryVerbosity: DD_IAST_TELEMETRY_VERBOSITY
     }
 
     this.isCiVisibility = isTrue(DD_IS_CIVISIBILITY)

package/packages/dd-trace/src/constants.js

@@ -28,6 +28,7 @@ module.exports = {
   CLIENT_PORT_KEY: 'network.destination.port',
   PEER_SERVICE_KEY: 'peer.service',
   PEER_SERVICE_SOURCE_KEY: '_dd.peer.service.source',
+  PEER_SERVICE_REMAP_KEY: '_dd.peer.service.remapped_from',
   SCI_REPOSITORY_URL: '_dd.git.repository_url',
   SCI_COMMIT_SHA: '_dd.git.commit.sha'
 }

package/packages/dd-trace/src/opentracing/tracer.js

@@ -27,6 +27,7 @@ class DatadogTracer {
     this._env = config.env
     this._tags = config.tags
     this._computePeerService = config.spanComputePeerService
+    this._peerServiceMapping = config.peerServiceMapping
     this._logInjection = config.logInjection
     this._debug = config.debug
     this._prioritySampler = new PrioritySampler(config.env, config.sampler)

package/packages/dd-trace/src/plugins/ci_plugin.js

@@ -111,7 +111,12 @@ module.exports = class CiPlugin extends Plugin {
     const childOf = getTestParentSpan(this.tracer)
 
     let testTags = {
-      ...getTestCommonTags(testName, testSuite, this.frameworkVersion),
+      ...getTestCommonTags(
+        testName,
+        testSuite,
+        this.frameworkVersion,
+        this.constructor.id
+      ),
       [COMPONENT]: this.constructor.id,
       ...extraTags
     }

package/packages/dd-trace/src/plugins/outbound.js

@@ -3,7 +3,8 @@
 const {
   CLIENT_PORT_KEY,
   PEER_SERVICE_KEY,
-  PEER_SERVICE_SOURCE_KEY
+  PEER_SERVICE_SOURCE_KEY,
+  PEER_SERVICE_REMAP_KEY
 } = require('../constants')
 const TracingPlugin = require('./tracing')
 
@@ -34,9 +35,11 @@ class OutboundPlugin extends TracingPlugin {
      * - If `peer.service` was defined _before_ we compute it (for example in custom instrumentation),
      *   `_dd.peer.service.source`'s value is `peer.service`
      */
-
-    if (tags['peer.service'] !== undefined) {
-      return { [PEER_SERVICE_SOURCE_KEY]: 'peer.service' }
+    if (tags[PEER_SERVICE_KEY] !== undefined) {
+      return {
+        [PEER_SERVICE_KEY]: tags[PEER_SERVICE_KEY],
+        [PEER_SERVICE_SOURCE_KEY]: PEER_SERVICE_KEY
+      }
     }
 
     const sourceTags = [
@@ -52,23 +55,37 @@ class OutboundPlugin extends TracingPlugin {
         }
       }
     }
-    return {}
+    return undefined
   }
 
-  startSpan (name, options) {
-    const span = super.startSpan(name, options)
-    return span
+  getPeerServiceRemap (peerData) {
+    /**
+     * If DD_TRACE_PEER_SERVICE_MAPPING is matched, we need to override the existing
+     * peer service and add the value we overrode.
+     */
+    const peerService = peerData[PEER_SERVICE_KEY]
+    if (peerService && this.tracer._peerServiceMapping[peerService]) {
+      return {
+        ...peerData,
+        [PEER_SERVICE_KEY]: this.tracer._peerServiceMapping[peerService],
+        [PEER_SERVICE_REMAP_KEY]: peerService
+      }
+    }
+    return peerData
   }
 
   finish () {
-    const span = this.activeSpan
+    this.tagPeerService(this.activeSpan)
+    super.finish(...arguments)
+  }
+
+  tagPeerService (span) {
     if (this.tracer._computePeerService) {
       const peerData = this.getPeerService(span.context()._tags)
-      if (peerData) {
-        span.addTags(peerData)
+      if (peerData !== undefined) {
+        span.addTags(this.getPeerServiceRemap(peerData))
       }
     }
-    super.finish(...arguments)
   }
 
   connect (url) {

package/packages/dd-trace/src/plugins/plugin.js

@@ -25,9 +25,31 @@ class Subscription {
   }
 }
 
+class StoreBinding {
+  constructor (event, transform) {
+    this._channel = dc.channel(event)
+    this._transform = data => {
+      const store = storage.getStore()
+
+      return !store || !store.noop
+        ? transform(data)
+        : store
+    }
+  }
+
+  enable () {
+    this._channel.bindStore(storage, this._transform)
+  }
+
+  disable () {
+    this._channel.unbindStore(storage, this._transform)
+  }
+}
+
 module.exports = class Plugin {
   constructor (tracer, tracerConfig) {
     this._subscriptions = []
+    this._bindings = []
     this._enabled = false
     this._tracer = tracer
     this.config = {} // plugin-specific configuration, unset until .configure() is called
@@ -53,6 +75,10 @@ module.exports = class Plugin {
     this._subscriptions.push(new Subscription(channelName, handler))
   }
 
+  addBind (channelName, transform) {
+    this._bindings.push(new StoreBinding(channelName, transform))
+  }
+
   addError (error) {
     const store = storage.getStore()
 
@@ -71,9 +97,11 @@ module.exports = class Plugin {
     if (config.enabled && !this._enabled) {
       this._enabled = true
       this._subscriptions.forEach(sub => sub.enable())
+      this._bindings.forEach(sub => sub.enable())
     } else if (!config.enabled && this._enabled) {
       this._enabled = false
       this._subscriptions.forEach(sub => sub.disable())
+      this._bindings.forEach(sub => sub.disable())
     }
   }
 }

package/packages/dd-trace/src/plugins/tracing.js

@@ -13,17 +13,7 @@ class TracingPlugin extends Plugin {
     this.component = this.constructor.component || this.constructor.id
     this.operation = this.constructor.operation
 
-    this.addTraceSub('start', message => {
-      this.start(message)
-    })
-
-    this.addTraceSub('error', err => {
-      this.error(err)
-    })
-
-    this.addTraceSub('finish', message => {
-      this.finish(message)
-    })
+    this.addTraceSubs()
   }
 
   get activeSpan () {
@@ -65,19 +55,43 @@ class TracingPlugin extends Plugin {
     this.addError(error)
   }
 
+  addTraceSubs () {
+    const events = ['start', 'end', 'asyncStart', 'asyncEnd', 'error', 'finish']
+
+    for (const event of events) {
+      const bindName = `bind${event.charAt(0).toUpperCase()}${event.slice(1)}`
+
+      if (this[event]) {
+        this.addTraceSub(event, message => {
+          this[event](message)
+        })
+      }
+
+      if (this[bindName]) {
+        this.addTraceBind(event, message => this[bindName](message))
+      }
+    }
+  }
+
   addTraceSub (eventName, handler) {
-    this.addSub(`apm:${this.component}:${this.operation}:${eventName}`, handler)
+    const prefix = this.constructor.prefix || `apm:${this.component}:${this.operation}`
+    this.addSub(`${prefix}:${eventName}`, handler)
   }
 
-  addError (error) {
-    const span = this.activeSpan
+  addTraceBind (eventName, transform) {
+    const prefix = this.constructor.prefix || `apm:${this.component}:${this.operation}`
+    this.addBind(`${prefix}:${eventName}`, transform)
+  }
 
+  addError (error, span = this.activeSpan) {
     if (!span._spanContext._tags['error']) {
+      // Errors may be wrapped in a context.
+      error = (error && error.error) || error
       span.setTag('error', error || 1)
     }
   }
 
-  startSpan (name, { childOf, kind, meta, metrics, service, resource, type } = {}) {
+  startSpan (name, { childOf, kind, meta, metrics, service, resource, type } = {}, enter = true) {
     const store = storage.getStore()
 
     if (store && childOf === undefined) {
@@ -100,7 +114,10 @@ class TracingPlugin extends Plugin {
 
     analyticsSampler.sample(span, this.config.measured)
 
-    storage.enterWith({ ...store, span })
+    // TODO: Remove this after migration to TracingChannel is done.
+    if (enter) {
+      storage.enterWith({ ...store, span })
+    }
 
     return span
   }

package/packages/dd-trace/src/plugins/util/ci.js

@@ -77,7 +77,7 @@ function filterSensitiveInfoFromRepository (repositoryUrl) {
 
     return `${protocol}//${hostname}${pathname}`
   } catch (e) {
-    return repositoryUrl
+    return ''
   }
 }
 

package/packages/dd-trace/src/plugins/util/test.js

@@ -1,11 +1,13 @@
 const path = require('path')
 const fs = require('fs')
+const { URL } = require('url')
+const log = require('../../log')
 
 const istanbul = require('istanbul-lib-coverage')
 const ignore = require('ignore')
 
 const { getGitMetadata } = require('./git')
-const { getUserProviderGitMetadata } = require('./user-provided-git')
+const { getUserProviderGitMetadata, validateGitRepositoryUrl, validateGitCommitSha } = require('./user-provided-git')
 const { getCIMetadata } = require('./ci')
 const { getRuntimeAndOSMetadata } = require('./env')
 const {
@@ -16,7 +18,8 @@ const {
   GIT_COMMIT_AUTHOR_EMAIL,
   GIT_COMMIT_AUTHOR_NAME,
   GIT_COMMIT_MESSAGE,
-  CI_WORKSPACE_PATH
+  CI_WORKSPACE_PATH,
+  CI_PIPELINE_URL
 } = require('./tags')
 const id = require('../../id')
 
@@ -104,7 +107,8 @@ module.exports = {
   mergeCoverage,
   fromCoverageMapToCoverage,
   getTestLineStart,
-  getCallSites
+  getCallSites,
+  removeInvalidMetadata
 }
 
 // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19
@@ -116,6 +120,39 @@ function getPkgManager () {
   }
 }
 
+function validateUrl (url) {
+  try {
+    const urlObject = new URL(url)
+    return (urlObject.protocol === 'https:' || urlObject.protocol === 'http:')
+  } catch (e) {
+    return false
+  }
+}
+
+function removeInvalidMetadata (metadata) {
+  return Object.keys(metadata).reduce((filteredTags, tag) => {
+    if (tag === GIT_REPOSITORY_URL) {
+      if (!validateGitRepositoryUrl(metadata[GIT_REPOSITORY_URL])) {
+        log.error('DD_GIT_REPOSITORY_URL must be a valid URL')
+        return filteredTags
+      }
+    }
+    if (tag === GIT_COMMIT_SHA) {
+      if (!validateGitCommitSha(metadata[GIT_COMMIT_SHA])) {
+        log.error('DD_GIT_COMMIT_SHA must be a full-length git SHA')
+        return filteredTags
+      }
+    }
+    if (tag === CI_PIPELINE_URL) {
+      if (!validateUrl(metadata[CI_PIPELINE_URL])) {
+        return filteredTags
+      }
+    }
+    filteredTags[tag] = metadata[tag]
+    return filteredTags
+  }, {})
+}
+
 function getTestEnvironmentMetadata (testFramework, config) {
   // TODO: eventually these will come from the tracer (generally available)
   const ciMetadata = getCIMetadata()
@@ -155,7 +192,7 @@ function getTestEnvironmentMetadata (testFramework, config) {
   if (config && config.service) {
     metadata['service.name'] = config.service
   }
-  return metadata
+  return removeInvalidMetadata(metadata)
 }
 
 function getTestParametersString (parametersByTestName, testName) {
@@ -173,6 +210,13 @@ function getTestParametersString (parametersByTestName, testName) {
   }
 }
 
+function getTestTypeFromFramework (testFramework) {
+  if (testFramework === 'playwright' || testFramework === 'cypress') {
+    return 'browser'
+  }
+  return 'test'
+}
+
 function finishAllTraceSpans (span) {
   span.context()._trace.started.forEach(traceSpan => {
     if (traceSpan !== span) {
@@ -188,10 +232,10 @@ function getTestParentSpan (tracer) {
   })
 }
 
-function getTestCommonTags (name, suite, version) {
+function getTestCommonTags (name, suite, version, testFramework) {
   return {
     [SPAN_TYPE]: 'test',
-    [TEST_TYPE]: 'test',
+    [TEST_TYPE]: getTestTypeFromFramework(testFramework),
     [SAMPLING_RULE_DECISION]: 1,
     [SAMPLING_PRIORITY]: AUTO_KEEP,
     [TEST_NAME]: name,
@@ -269,12 +313,12 @@ function getCodeOwnersForFilename (filename, entries) {
   return null
 }
 
-function getTestLevelCommonTags (command, testFrameworkVersion) {
+function getTestLevelCommonTags (command, testFrameworkVersion, testFramework) {
   return {
     [TEST_FRAMEWORK_VERSION]: testFrameworkVersion,
     [LIBRARY_VERSION]: ddTraceVersion,
     [TEST_COMMAND]: command,
-    [TEST_TYPE]: 'test'
+    [TEST_TYPE]: getTestTypeFromFramework(testFramework)
   }
 }
 
@@ -284,7 +328,7 @@ function getTestSessionCommonTags (command, testFrameworkVersion, testFramework)
     [RESOURCE_NAME]: `test_session.${command}`,
     [TEST_MODULE]: testFramework,
     [TEST_TOOLCHAIN]: getPkgManager(),
-    ...getTestLevelCommonTags(command, testFrameworkVersion)
+    ...getTestLevelCommonTags(command, testFrameworkVersion, testFramework)
   }
 }
 
@@ -293,7 +337,7 @@ function getTestModuleCommonTags (command, testFrameworkVersion, testFramework)
     [SPAN_TYPE]: 'test_module_end',
     [RESOURCE_NAME]: `test_module.${command}`,
     [TEST_MODULE]: testFramework,
-    ...getTestLevelCommonTags(command, testFrameworkVersion)
+    ...getTestLevelCommonTags(command, testFrameworkVersion, testFramework)
   }
 }
 
@@ -303,7 +347,7 @@ function getTestSuiteCommonTags (command, testFrameworkVersion, testSuite, testF
     [RESOURCE_NAME]: `test_suite.${testSuite}`,
     [TEST_MODULE]: testFramework,
     [TEST_SUITE]: testSuite,
-    ...getTestLevelCommonTags(command, testFrameworkVersion)
+    ...getTestLevelCommonTags(command, testFrameworkVersion, testFramework)
   }
 }