dd-trace 5.38.0 → 5.40.0

package/packages/dd-trace/src/telemetry/telemetry.js

@@ -0,0 +1,394 @@
+'use strict'
+const tracerVersion = require('../../../../package.json').version
+const dc = require('dc-polyfill')
+const os = require('os')
+const dependencies = require('./dependencies')
+const { sendData } = require('./send-data')
+const { errors } = require('../startup-log')
+const { manager: metricsManager } = require('./metrics')
+const telemetryLogger = require('./logs')
+const logger = require('../log')
+
+const telemetryStartChannel = dc.channel('datadog:telemetry:start')
+const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
+const telemetryAppClosingChannel = dc.channel('datadog:telemetry:app-closing')
+
+let config
+let pluginManager
+
+let application
+let host
+let heartbeatTimeout
+let heartbeatInterval
+let extendedInterval
+let integrations
+let configWithOrigin = []
+let retryData = null
+const extendedHeartbeatPayload = {}
+
+const sentIntegrations = new Set()
+
+function getRetryData () {
+  return retryData
+}
+
+function updateRetryData (error, retryObj) {
+  if (error) {
+    if (retryObj.reqType === 'message-batch') {
+      const payload = retryObj.payload[0].payload
+      const reqType = retryObj.payload[0].request_type
+      retryData = { payload, reqType }
+
+      // Since this payload failed twice it now gets save in to the extended heartbeat
+      const failedPayload = retryObj.payload[1].payload
+      const failedReqType = retryObj.payload[1].request_type
+
+      // save away the dependencies and integration request for extended heartbeat.
+      if (failedReqType === 'app-integrations-change') {
+        if (extendedHeartbeatPayload.integrations) {
+          extendedHeartbeatPayload.integrations.push(failedPayload)
+        } else {
+          extendedHeartbeatPayload.integrations = [failedPayload]
+        }
+      }
+      if (failedReqType === 'app-dependencies-loaded') {
+        if (extendedHeartbeatPayload.dependencies) {
+          extendedHeartbeatPayload.dependencies.push(failedPayload)
+        } else {
+          extendedHeartbeatPayload.dependencies = [failedPayload]
+        }
+      }
+    } else {
+      retryData = retryObj
+    }
+  } else {
+    retryData = null
+  }
+}
+
+function getIntegrations () {
+  const newIntegrations = []
+  for (const pluginName in pluginManager._pluginsByName) {
+    if (sentIntegrations.has(pluginName)) {
+      continue
+    }
+    newIntegrations.push({
+      name: pluginName,
+      enabled: pluginManager._pluginsByName[pluginName]._enabled,
+      auto_enabled: true
+    })
+    sentIntegrations.add(pluginName)
+  }
+  return newIntegrations
+}
+
+function getProducts (config) {
+  const products = {
+    appsec: {
+      enabled: config.appsec.enabled
+    },
+    profiler: {
+      version: tracerVersion,
+      enabled: profilingEnabledToBoolean(config.profiling.enabled)
+    }
+  }
+  if (errors.profilingError) {
+    products.profiler.error = errors.profilingError
+    errors.profilingError = {}
+  }
+  return products
+}
+
+function getInstallSignature (config) {
+  const { installSignature: sig } = config
+  if (sig && (sig.id || sig.time || sig.type)) {
+    return {
+      install_id: sig.id,
+      install_time: sig.time,
+      install_type: sig.type
+    }
+  }
+}
+
+function appStarted (config) {
+  const app = {
+    products: getProducts(config),
+    configuration: configWithOrigin
+  }
+  const installSignature = getInstallSignature(config)
+  if (installSignature) {
+    app.install_signature = installSignature
+  }
+  // TODO: add app.error with correct error codes
+  // if (errors.agentError) {
+  //   app.error = errors.agentError
+  //   errors.agentError = {}
+  // }
+  return app
+}
+
+function appClosing () {
+  if (!config?.telemetry?.enabled) {
+    return
+  }
+  // Give chance to listeners to update metrics before shutting down.
+  telemetryAppClosingChannel.publish()
+  const { reqType, payload } = createPayload('app-closing')
+  sendData(config, application, host, reqType, payload)
+  // We flush before shutting down.
+  metricsManager.send(config, application, host)
+  telemetryLogger.send(config, application, host)
+}
+
+function onBeforeExit () {
+  process.removeListener('beforeExit', onBeforeExit)
+  appClosing()
+}
+
+function createAppObject (config) {
+  return {
+    service_name: config.service,
+    env: config.env,
+    service_version: config.version,
+    tracer_version: tracerVersion,
+    language_name: 'nodejs',
+    language_version: process.versions.node
+  }
+}
+
+function createHostObject () {
+  const osName = os.type()
+
+  if (osName === 'Linux' || osName === 'Darwin') {
+    return {
+      hostname: os.hostname(),
+      os: osName,
+      architecture: os.arch(),
+      kernel_version: os.version(),
+      kernel_release: os.release(),
+      kernel_name: osName
+    }
+  }
+
+  if (osName === 'Windows_NT') {
+    return {
+      hostname: os.hostname(),
+      os: osName,
+      architecture: os.arch(),
+      os_version: os.version()
+    }
+  }
+
+  return {
+    hostname: os.hostname(), // TODO is this enough?
+    os: osName
+  }
+}
+
+function getTelemetryData () {
+  return { config, application, host, heartbeatInterval }
+}
+
+function createBatchPayload (payload) {
+  const batchPayload = payload.map(item => {
+    return {
+      request_type: item.reqType,
+      payload: item.payload
+    }
+  })
+
+  return batchPayload
+}
+
+function createPayload (currReqType, currPayload = {}) {
+  if (getRetryData()) {
+    const payload = { reqType: currReqType, payload: currPayload }
+    const batchPayload = createBatchPayload([payload, retryData])
+    return { reqType: 'message-batch', payload: batchPayload }
+  }
+
+  return { reqType: currReqType, payload: currPayload }
+}
+
+function heartbeat (config, application, host) {
+  heartbeatTimeout = setTimeout(() => {
+    metricsManager.send(config, application, host)
+    telemetryLogger.send(config, application, host)
+
+    const { reqType, payload } = createPayload('app-heartbeat')
+    sendData(config, application, host, reqType, payload, updateRetryData)
+    heartbeat(config, application, host)
+  }, heartbeatInterval).unref()
+  return heartbeatTimeout
+}
+
+function extendedHeartbeat (config) {
+  extendedInterval = setInterval(() => {
+    const appPayload = appStarted(config)
+    const payload = {
+      ...appPayload,
+      ...extendedHeartbeatPayload
+    }
+    sendData(config, application, host, 'app-extended-heartbeat', payload)
+    Object.keys(extendedHeartbeatPayload).forEach(key => delete extendedHeartbeatPayload[key])
+  }, 1000 * 60 * 60 * 24).unref()
+  return extendedInterval
+}
+
+function start (aConfig, thePluginManager) {
+  if (!aConfig.telemetry.enabled) {
+    if (aConfig.sca?.enabled) {
+      logger.warn('DD_APPSEC_SCA_ENABLED requires enabling telemetry to work.')
+    }
+
+    return
+  }
+  config = aConfig
+  pluginManager = thePluginManager
+  application = createAppObject(config)
+  host = createHostObject()
+  heartbeatInterval = config.telemetry.heartbeatInterval
+  integrations = getIntegrations()
+
+  dependencies.start(config, application, host, getRetryData, updateRetryData)
+  telemetryLogger.start(config)
+
+  sendData(config, application, host, 'app-started', appStarted(config))
+
+  if (integrations.length > 0) {
+    sendData(config, application, host, 'app-integrations-change',
+      { integrations }, updateRetryData)
+  }
+
+  heartbeat(config, application, host)
+
+  extendedHeartbeat(config)
+
+  process.on('beforeExit', onBeforeExit)
+  telemetryStartChannel.publish(getTelemetryData())
+}
+
+function stop () {
+  if (!config) {
+    return
+  }
+  clearInterval(extendedInterval)
+  clearTimeout(heartbeatTimeout)
+  process.removeListener('beforeExit', onBeforeExit)
+
+  telemetryStopChannel.publish(getTelemetryData())
+
+  config = undefined
+}
+
+function updateIntegrations () {
+  if (!config || !config.telemetry.enabled) {
+    return
+  }
+  const integrations = getIntegrations()
+  if (integrations.length === 0) {
+    return
+  }
+
+  const { reqType, payload } = createPayload('app-integrations-change', { integrations })
+
+  sendData(config, application, host, reqType, payload, updateRetryData)
+}
+
+function formatMapForTelemetry (map) {
+  // format from an object to a string map in order for
+  // telemetry intake to accept the configuration
+  return map
+    ? Object.entries(map).map(([key, value]) => `${key}:${value}`).join(',')
+    : ''
+}
+
+function updateConfig (changes, config) {
+  if (!config.telemetry.enabled) return
+  if (changes.length === 0) return
+
+  logger.trace(changes)
+
+  const application = createAppObject(config)
+  const host = createHostObject()
+
+  const nameMapping = {
+    sampleRate: 'DD_TRACE_SAMPLE_RATE',
+    logInjection: 'DD_LOG_INJECTION',
+    headerTags: 'DD_TRACE_HEADER_TAGS',
+    tags: 'DD_TAGS',
+    'sampler.rules': 'DD_TRACE_SAMPLING_RULES',
+    traceEnabled: 'DD_TRACE_ENABLED',
+    url: 'DD_TRACE_AGENT_URL',
+    'sampler.rateLimit': 'DD_TRACE_RATE_LIMIT',
+    queryStringObfuscation: 'DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP',
+    version: 'DD_VERSION',
+    env: 'DD_ENV',
+    service: 'DD_SERVICE',
+    clientIpHeader: 'DD_TRACE_CLIENT_IP_HEADER',
+    'grpc.client.error.statuses': 'DD_GRPC_CLIENT_ERROR_STATUSES',
+    'grpc.server.error.statuses': 'DD_GRPC_SERVER_ERROR_STATUSES',
+    traceId128BitLoggingEnabled: 'DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED'
+  }
+
+  const namesNeedFormatting = new Set(['DD_TAGS', 'peerServiceMapping', 'serviceMapping'])
+
+  const configuration = []
+  const names = [] // list of config names whose values have been changed
+
+  for (const change of changes) {
+    const name = nameMapping[change.name] || change.name
+
+    names.push(name)
+    const { origin, value } = change
+    const entry = { name, value, origin }
+
+    if (namesNeedFormatting.has(entry.name)) {
+      entry.value = formatMapForTelemetry(entry.value)
+    } else if (entry.name === 'url') {
+      if (entry.value) {
+        entry.value = entry.value.toString()
+      }
+    } else if (entry.name === 'DD_TRACE_SAMPLING_RULES') {
+      entry.value = JSON.stringify(entry.value)
+    } else if (Array.isArray(entry.value)) {
+      entry.value = value.join(',')
+    }
+    configuration.push(entry)
+  }
+
+  function isNotModified (entry) {
+    return !names.includes(entry.name)
+  }
+
+  if (!configWithOrigin.length) {
+    configWithOrigin = configuration
+  } else {
+    // update configWithOrigin to contain up-to-date full list of config values for app-extended-heartbeat
+    configWithOrigin = configWithOrigin.filter(isNotModified)
+    configWithOrigin = configWithOrigin.concat(configuration)
+    const { reqType, payload } = createPayload('app-client-configuration-change', { configuration })
+    sendData(config, application, host, reqType, payload, updateRetryData)
+  }
+}
+
+function profilingEnabledToBoolean (profilingEnabled) {
+  if (typeof profilingEnabled === 'boolean') {
+    return profilingEnabled
+  }
+  if (['auto', 'true'].includes(profilingEnabled)) {
+    return true
+  }
+  if (profilingEnabled === 'false') {
+    return false
+  }
+  return undefined
+}
+
+module.exports = {
+  start,
+  stop,
+  updateIntegrations,
+  updateConfig,
+  appClosing
+}

package/packages/dd-trace/src/tracer.js

@@ -6,10 +6,7 @@ const Scope = require('./scope')
 const { isError } = require('./util')
 const { setStartupLogConfig } = require('./startup-log')
 const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants')
-const { DataStreamsProcessor } = require('./datastreams/processor')
-const { DsmPathwayCodec } = require('./datastreams/pathway')
-const DataStreamsContext = require('./data_streams_context')
-const { DataStreamsCheckpointer } = require('./data_streams')
+const { DataStreamsCheckpointer, DataStreamsManager, DataStreamsProcessor } = require('./datastreams')
 const { flushStartupLogs } = require('../../datadog-instrumentations/src/check_require_cache')
 const log = require('./log/writer')
 
@@ -22,31 +19,26 @@ class DatadogTracer extends Tracer {
   constructor (config, prioritySampler) {
     super(config, prioritySampler)
     this._dataStreamsProcessor = new DataStreamsProcessor(config)
+    this._dataStreamsManager = new DataStreamsManager(this._dataStreamsProcessor)
     this.dataStreamsCheckpointer = new DataStreamsCheckpointer(this)
     this._scope = new Scope()
     setStartupLogConfig(config)
     flushStartupLogs(log)
   }
 
-  configure ({ env, sampler }) {
-    this._prioritySampler.configure(env, sampler)
+  configure (config) {
+    const { env, sampler } = config
+    this._prioritySampler.configure(env, sampler, config)
   }
 
   // todo[piochelepiotr] These two methods are not related to the tracer, but to data streams monitoring.
   // They should be moved outside of the tracer in the future.
   setCheckpoint (edgeTags, span, payloadSize = 0) {
-    const ctx = this._dataStreamsProcessor.setCheckpoint(
-      edgeTags, span, DataStreamsContext.getDataStreamsContext(), payloadSize
-    )
-    DataStreamsContext.setDataStreamsContext(ctx)
-    return ctx
+    return this._dataStreamsManager.setCheckpoint(edgeTags, span, payloadSize)
   }
 
   decodeDataStreamsContext (carrier) {
-    const ctx = DsmPathwayCodec.decode(carrier)
-    // we erase the previous context everytime we decode a new one
-    DataStreamsContext.setDataStreamsContext(ctx)
-    return ctx
+    return this._dataStreamsManager.decodeDataStreamsContext(carrier)
   }
 
   setOffset (offsetData) {

package/packages/dd-trace/src/appsec/standalone.js

@@ -1,130 +0,0 @@
-'use strict'
-
-const { channel } = require('dc-polyfill')
-const { USER_KEEP, AUTO_KEEP, AUTO_REJECT } = require('../../../../ext/priority')
-const { MANUAL_KEEP } = require('../../../../ext/tags')
-const PrioritySampler = require('../priority_sampler')
-const RateLimiter = require('../rate_limiter')
-const TraceState = require('../opentracing/propagation/tracestate')
-const { hasOwn } = require('../util')
-const { APM_TRACING_ENABLED_KEY, APPSEC_PROPAGATION_KEY, SAMPLING_MECHANISM_DEFAULT } = require('../constants')
-
-const startCh = channel('dd-trace:span:start')
-const injectCh = channel('dd-trace:span:inject')
-const extractCh = channel('dd-trace:span:extract')
-
-let enabled
-
-class StandAloneAsmPrioritySampler extends PrioritySampler {
-  constructor (env) {
-    super(env, { sampleRate: 0, rateLimit: 0, rules: [] })
-
-    // let some regular APM traces go through, 1 per minute to keep alive the service
-    this._limiter = new RateLimiter(1, 'minute')
-  }
-
-  configure (env, config) {
-    // rules not supported
-    this._env = env
-  }
-
-  _getPriorityFromTags (tags, context) {
-    if (hasOwn(tags, MANUAL_KEEP) &&
-      tags[MANUAL_KEEP] !== false &&
-      hasOwn(context._trace.tags, APPSEC_PROPAGATION_KEY)
-    ) {
-      return USER_KEEP
-    }
-  }
-
-  _getPriorityFromAuto (span) {
-    const context = this._getContext(span)
-
-    context._sampling.mechanism = SAMPLING_MECHANISM_DEFAULT
-
-    if (hasOwn(context._trace.tags, APPSEC_PROPAGATION_KEY)) {
-      return USER_KEEP
-    }
-
-    return this._isSampledByRateLimit(context) ? AUTO_KEEP : AUTO_REJECT
-  }
-}
-
-function onSpanStart ({ span, fields }) {
-  const tags = span.context?.()?._tags
-  if (!tags) return
-
-  const { parent } = fields
-  if (!parent || parent._isRemote) {
-    tags[APM_TRACING_ENABLED_KEY] = 0
-  }
-}
-
-function onSpanInject ({ spanContext, carrier }) {
-  if (!spanContext?._trace?.tags || !carrier) return
-
-  // do not inject trace and sampling if there is no appsec event
-  if (!hasOwn(spanContext._trace.tags, APPSEC_PROPAGATION_KEY)) {
-    for (const key in carrier) {
-      const lKey = key.toLowerCase()
-      if (lKey.startsWith('x-datadog')) {
-        delete carrier[key]
-      } else if (lKey === 'tracestate') {
-        const tracestate = TraceState.fromString(carrier[key])
-        tracestate.forVendor('dd', state => state.clear())
-        carrier[key] = tracestate.toString()
-      }
-    }
-  }
-}
-
-function onSpanExtract ({ spanContext = {} }) {
-  if (!spanContext._trace?.tags || !spanContext._sampling) return
-
-  // reset upstream priority if _dd.p.appsec is not found
-  if (!hasOwn(spanContext._trace.tags, APPSEC_PROPAGATION_KEY)) {
-    spanContext._sampling.priority = undefined
-  } else if (spanContext._sampling.priority !== USER_KEEP) {
-    spanContext._sampling.priority = USER_KEEP
-  }
-}
-
-function sample (span) {
-  const spanContext = span.context?.()
-  if (enabled && spanContext?._trace?.tags) {
-    spanContext._trace.tags[APPSEC_PROPAGATION_KEY] = '1'
-
-    // TODO: ask. can we reset here sampling like this?
-    if (spanContext._sampling?.priority < AUTO_KEEP) {
-      spanContext._sampling.priority = undefined
-    }
-  }
-}
-
-function configure (config) {
-  const configChanged = enabled !== config.appsec?.standalone?.enabled
-  if (!configChanged) return
-
-  enabled = config.appsec?.standalone?.enabled
-
-  let prioritySampler
-  if (enabled) {
-    startCh.subscribe(onSpanStart)
-    injectCh.subscribe(onSpanInject)
-    extractCh.subscribe(onSpanExtract)
-
-    prioritySampler = new StandAloneAsmPrioritySampler(config.env)
-  } else {
-    if (startCh.hasSubscribers) startCh.unsubscribe(onSpanStart)
-    if (injectCh.hasSubscribers) injectCh.unsubscribe(onSpanInject)
-    if (extractCh.hasSubscribers) extractCh.unsubscribe(onSpanExtract)
-  }
-
-  return prioritySampler
-}
-
-module.exports = {
-  configure,
-  sample,
-  StandAloneAsmPrioritySampler
-}