dd-trace 4.18.0 → 4.22.0

package/packages/dd-trace/src/spanleak.js

@@ -0,0 +1,98 @@
+'use strict'
+
+/* eslint-disable no-console */
+
+const SortedSet = require('tlhunter-sorted-set')
+
+const INTERVAL = 1000 // look for expired spans every 1s
+const LIFETIME = 60 * 1000 // all spans have a max lifetime of 1m
+
+const MODES = {
+  DISABLED: 0,
+  // METRICS_ONLY
+  LOG: 1,
+  GC_AND_LOG: 2
+  // GC
+}
+
+module.exports.MODES = MODES
+
+const spans = new SortedSet()
+
+// TODO: should these also be delivered as runtime metrics?
+
+// const registry = new FinalizationRegistry(name => {
+//   spans.del(span) // there is no span
+// })
+
+let interval
+let mode = MODES.DISABLED
+
+module.exports.disable = function () {
+  mode = MODES.DISABLED
+}
+
+module.exports.enableLogging = function () {
+  mode = MODES.LOG
+}
+
+module.exports.enableGarbageCollection = function () {
+  mode = MODES.GC_AND_LOG
+}
+
+module.exports.startScrubber = function () {
+  if (!isEnabled()) return
+
+  interval = setInterval(() => {
+    const now = Date.now()
+    const expired = spans.rangeByScore(0, now)
+
+    if (!expired.length) return
+
+    const gc = isGarbageCollecting()
+
+    const expirationsByType = Object.create(null) // { [spanType]: count }
+
+    for (const wrapped of expired) {
+      spans.del(wrapped)
+      const span = wrapped.deref()
+
+      if (!span) continue // span has already been garbage collected
+
+      // TODO: Should we also do things like record the route to help users debug leaks?
+      if (!expirationsByType[span._name]) expirationsByType[span._name] = 0
+      expirationsByType[span._name]++
+
+      if (!gc) continue // everything after this point is related to manual GC
+
+      // TODO: what else can we do to alleviate memory usage
+      span.context()._tags = Object.create(null)
+    }
+
+    console.log('expired spans:' +
+      Object.keys(expirationsByType).reduce((a, c) => `${a} ${c}: ${expirationsByType[c]}`, ''))
+  }, INTERVAL)
+}
+
+module.exports.stopScrubber = function () {
+  clearInterval(interval)
+}
+
+module.exports.addSpan = function (span) {
+  if (!isEnabled()) return
+
+  const now = Date.now()
+  const expiration = now + LIFETIME
+  // eslint-disable-next-line no-undef
+  const wrapped = new WeakRef(span)
+  spans.add(wrapped, expiration)
+  // registry.register(span, span._name)
+}
+
+function isEnabled () {
+  return mode > MODES.DISABLED
+}
+
+function isGarbageCollecting () {
+  return mode >= MODES.GC_AND_LOG
+}

package/packages/dd-trace/src/startup-log.js

@@ -6,6 +6,7 @@ const os = require('os')
 const { inspect } = require('util')
 const tracerVersion = require('../../../package.json').version
 
+const errors = {}
 let config
 let pluginManager
 let samplingRules = []
@@ -89,6 +90,10 @@ function startupLog ({ agentError } = {}) {
   info('DATADOG TRACER CONFIGURATION - ' + out)
   if (agentError) {
     warn('DATADOG TRACER DIAGNOSTIC - Agent Error: ' + agentError.message)
+    errors.agentError = {
+      code: agentError.code ? agentError.code : '',
+      message: `Agent Error:${agentError.message}`
+    }
   }
 
   config = undefined
@@ -112,5 +117,6 @@ module.exports = {
   startupLog,
   setStartupLogConfig,
   setStartupLogPluginManager,
-  setSamplingRules
+  setSamplingRules,
+  errors
 }

package/packages/dd-trace/src/telemetry/dependencies.js

@@ -4,8 +4,9 @@ const path = require('path')
 const parse = require('module-details-from-path')
 const requirePackageJson = require('../require-package-json')
 const { sendData } = require('./send-data')
-const dc = require('../../../diagnostics_channel')
+const dc = require('dc-polyfill')
 const { fileURLToPath } = require('url')
+const { isTrue } = require('../../src/util')
 
 const savedDependenciesToSend = new Set()
 const detectedDependencyKeys = new Set()
@@ -14,20 +15,57 @@ const detectedDependencyVersions = new Set()
 const FILE_URI_START = `file://`
 const moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart')
 
-let immediate, config, application, host
+let immediate, config, application, host, initialLoad
 let isFirstModule = true
+let getRetryData
+let updateRetryData
 
+function createBatchPayload (payload) {
+  const batchPayload = []
+  payload.map(item => {
+    batchPayload.push({
+      request_type: item.reqType,
+      payload: item.payload
+    })
+  })
+
+  return batchPayload
+}
 function waitAndSend (config, application, host) {
   if (!immediate) {
     immediate = setImmediate(() => {
       immediate = null
       if (savedDependenciesToSend.size > 0) {
-        const dependencies = Array.from(savedDependenciesToSend.values()).splice(0, 1000).map(pair => {
-          savedDependenciesToSend.delete(pair)
-          const [name, version] = pair.split(' ')
-          return { name, version }
-        })
-        sendData(config, application, host, 'app-dependencies-loaded', { dependencies })
+        const dependencies = Array.from(savedDependenciesToSend.values())
+          // if a depencdency is from the initial load, *always* send the event
+          // Otherwise, only send if dependencyCollection is enabled
+          .filter(dep => {
+            const initialLoadModule = isTrue(dep.split(' ')[2])
+            const sendModule = initialLoadModule || (config.telemetry?.dependencyCollection)
+
+            if (!sendModule) savedDependenciesToSend.delete(dep) // we'll never send it
+            return sendModule
+          })
+          .splice(0, 2000) // v2 documentation specifies up to 2000 dependencies can be sent at once
+          .map(pair => {
+            savedDependenciesToSend.delete(pair)
+            const [name, version] = pair.split(' ')
+            return { name, version }
+          })
+        let currPayload
+        const retryData = getRetryData()
+        if (retryData) {
+          currPayload = { reqType: 'app-dependencies-loaded', payload: { dependencies } }
+        } else {
+          if (!dependencies.length) return // no retry data and no dependencies, nothing to send
+          currPayload = { dependencies }
+        }
+
+        const payload = retryData ? createBatchPayload([currPayload, retryData]) : currPayload
+        const reqType = retryData ? 'message-batch' : 'app-dependencies-loaded'
+
+        sendData(config, application, host, reqType, payload, updateRetryData)
+
         if (savedDependenciesToSend.size > 0) {
           waitAndSend(config, application, host)
         }
@@ -76,7 +114,7 @@ function onModuleLoad (data) {
             const dependencyAndVersion = `${name} ${version}`
 
             if (!detectedDependencyVersions.has(dependencyAndVersion)) {
-              savedDependenciesToSend.add(dependencyAndVersion)
+              savedDependenciesToSend.add(`${dependencyAndVersion} ${initialLoad}`)
               detectedDependencyVersions.add(dependencyAndVersion)
 
               waitAndSend(config, application, host)
@@ -89,11 +127,19 @@ function onModuleLoad (data) {
     }
   }
 }
-function start (_config, _application, _host) {
+function start (_config = {}, _application, _host, getRetryDataFunction, updateRetryDatafunction) {
   config = _config
   application = _application
   host = _host
+  initialLoad = true
+  getRetryData = getRetryDataFunction
+  updateRetryData = updateRetryDatafunction
   moduleLoadStartChannel.subscribe(onModuleLoad)
+
+  // try and capture intially loaded modules in the first tick
+  // since, ideally, the tracer (and this module) should be loaded first,
+  // this should capture any first-tick dependencies
+  queueMicrotask(() => { initialLoad = false })
 }
 
 function isDependency (filename, request) {

package/packages/dd-trace/src/telemetry/index.js

@@ -1,13 +1,11 @@
 'use strict'
-
 const tracerVersion = require('../../../../package.json').version
-const dc = require('../../../diagnostics_channel')
+const dc = require('dc-polyfill')
 const os = require('os')
 const dependencies = require('./dependencies')
 const { sendData } = require('./send-data')
-
+const { errors } = require('../startup-log')
 const { manager: metricsManager } = require('./metrics')
-const logs = require('./logs')
 
 const telemetryStartChannel = dc.channel('datadog:telemetry:start')
 const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
@@ -17,11 +15,53 @@ let pluginManager
 
 let application
 let host
-let interval
 let heartbeatTimeout
 let heartbeatInterval
+let extendedInterval
+let integrations
+let retryData = null
+const extendedHeartbeatPayload = {}
+
 const sentIntegrations = new Set()
 
+function getRetryData () {
+  return retryData
+}
+
+function updateRetryData (error, retryObj) {
+  if (error) {
+    if (retryObj.reqType === 'message-batch') {
+      const payload = retryObj.payload[0].payload
+      const reqType = retryObj.payload[0].request_type
+      retryData = { payload: payload, reqType: reqType }
+
+      // Since this payload failed twice it now gets save in to the extended heartbeat
+      const failedPayload = retryObj.payload[1].payload
+      const failedReqType = retryObj.payload[1].request_type
+
+      // save away the dependencies and integration request for extended heartbeat.
+      if (failedReqType === 'app-integrations-change') {
+        if (extendedHeartbeatPayload['integrations']) {
+          extendedHeartbeatPayload['integrations'].push(failedPayload)
+        } else {
+          extendedHeartbeatPayload['integrations'] = [failedPayload]
+        }
+      }
+      if (failedReqType === 'app-dependencies-loaded') {
+        if (extendedHeartbeatPayload['dependencies']) {
+          extendedHeartbeatPayload['dependencies'].push(failedPayload)
+        } else {
+          extendedHeartbeatPayload['dependencies'] = [failedPayload]
+        }
+      }
+    } else {
+      retryData = retryObj
+    }
+  } else {
+    retryData = null
+  }
+}
+
 function getIntegrations () {
   const newIntegrations = []
   for (const pluginName in pluginManager._pluginsByName) {
@@ -38,6 +78,23 @@ function getIntegrations () {
   return newIntegrations
 }
 
+function getProducts (config) {
+  const products = {
+    appsec: {
+      enabled: config.appsec.enabled
+    },
+    profiler: {
+      version: tracerVersion,
+      enabled: config.profiling.enabled
+    }
+  }
+  if (errors.profilingError) {
+    products.profiler.error = errors.profilingError
+    errors.profilingError = {}
+  }
+  return products
+}
+
 function flatten (input, result = [], prefix = [], traversedObjects = null) {
   traversedObjects = traversedObjects || new WeakSet()
   if (traversedObjects.has(input)) {
@@ -48,33 +105,30 @@ function flatten (input, result = [], prefix = [], traversedObjects = null) {
     if (typeof value === 'object' && value !== null) {
       flatten(value, result, [...prefix, key], traversedObjects)
     } else {
-      result.push({ name: [...prefix, key].join('.'), value })
+      // TODO: add correct origin value
+      result.push({ name: [...prefix, key].join('.'), value, origin: 'unknown' })
     }
   }
   return result
 }
 
-function appStarted () {
-  return {
-    integrations: getIntegrations(),
-    dependencies: [],
-    configuration: flatten(formatConfig(config)),
-    additional_payload: []
+function appStarted (config) {
+  const app = {
+    products: getProducts(config),
+    configuration: flatten(config)
   }
-}
-
-function formatConfig (config) {
-  // format peerServiceMapping from an object to a string map in order for
-  // telemetry intake to accept the configuration
-  config.peerServiceMapping = config.peerServiceMapping
-    ? Object.entries(config.peerServiceMapping).map(([key, value]) => `${key}:${value}`).join(',')
-    : ''
-  return config
+  // TODO: add app.error with correct error codes
+  // if (errors.agentError) {
+  //   app.error = errors.agentError
+  //   errors.agentError = {}
+  // }
+  return app
 }
 
 function onBeforeExit () {
   process.removeListener('beforeExit', onBeforeExit)
-  sendData(config, application, host, 'app-closing')
+  const { reqType, payload } = createPayload('app-closing')
+  sendData(config, application, host, reqType, payload)
 }
 
 function createAppObject (config) {
@@ -121,14 +175,52 @@ function getTelemetryData () {
   return { config, application, host, heartbeatInterval }
 }
 
+function createBatchPayload (payload) {
+  const batchPayload = []
+  payload.map(item => {
+    batchPayload.push({
+      request_type: item.reqType,
+      payload: item.payload
+    })
+  })
+
+  return batchPayload
+}
+
+function createPayload (currReqType, currPayload = {}) {
+  if (getRetryData()) {
+    const payload = { reqType: currReqType, payload: currPayload }
+    const batchPayload = createBatchPayload([payload, retryData])
+    return { 'reqType': 'message-batch', 'payload': batchPayload }
+  }
+
+  return { 'reqType': currReqType, 'payload': currPayload }
+}
+
 function heartbeat (config, application, host) {
   heartbeatTimeout = setTimeout(() => {
-    sendData(config, application, host, 'app-heartbeat')
+    metricsManager.send(config, application, host)
+
+    const { reqType, payload } = createPayload('app-heartbeat')
+    sendData(config, application, host, reqType, payload, updateRetryData)
     heartbeat(config, application, host)
   }, heartbeatInterval).unref()
   return heartbeatTimeout
 }
 
+function extendedHeartbeat (config) {
+  extendedInterval = setInterval(() => {
+    const appPayload = appStarted(config)
+    const payload = {
+      ...appPayload,
+      ...extendedHeartbeatPayload
+    }
+    sendData(config, application, host, 'app-extended-heartbeat', payload)
+    Object.keys(extendedHeartbeatPayload).forEach(key => delete extendedHeartbeatPayload[key])
+  }, 1000 * 60 * 60 * 24).unref()
+  return extendedInterval
+}
+
 function start (aConfig, thePluginManager) {
   if (!aConfig.telemetry.enabled) {
     return
@@ -138,19 +230,22 @@ function start (aConfig, thePluginManager) {
   application = createAppObject(config)
   host = createHostObject()
   heartbeatInterval = config.telemetry.heartbeatInterval
+  integrations = getIntegrations()
+
+  dependencies.start(config, application, host, getRetryData, updateRetryData)
+
+  sendData(config, application, host, 'app-started', appStarted(config))
 
-  dependencies.start(config, application, host)
-  logs.start(config)
+  if (integrations.length > 0) {
+    sendData(config, application, host, 'app-integrations-change',
+      { integrations }, updateRetryData)
+  }
 
-  sendData(config, application, host, 'app-started', appStarted())
   heartbeat(config, application, host)
-  interval = setInterval(() => {
-    metricsManager.send(config, application, host)
-    logs.send(config, application, host)
-  }, heartbeatInterval)
-  interval.unref()
-  process.on('beforeExit', onBeforeExit)
 
+  extendedHeartbeat(config)
+
+  process.on('beforeExit', onBeforeExit)
   telemetryStartChannel.publish(getTelemetryData())
 }
 
@@ -158,7 +253,7 @@ function stop () {
   if (!config) {
     return
   }
-  clearInterval(interval)
+  clearInterval(extendedInterval)
   clearTimeout(heartbeatTimeout)
   process.removeListener('beforeExit', onBeforeExit)
 
@@ -175,7 +270,10 @@ function updateIntegrations () {
   if (integrations.length === 0) {
     return
   }
-  sendData(config, application, host, 'app-integrations-change', { integrations })
+
+  const { reqType, payload } = createPayload('app-integrations-change', { integrations })
+
+  sendData(config, application, host, reqType, payload, updateRetryData)
 }
 
 function updateConfig (changes, config) {
@@ -188,21 +286,15 @@ function updateConfig (changes, config) {
   const application = createAppObject(config)
   const host = createHostObject()
 
-  const names = {
-    sampleRate: 'DD_TRACE_SAMPLE_RATE',
-    logInjection: 'DD_LOG_INJECTION',
-    headerTags: 'DD_TRACE_HEADER_TAGS'
-  }
-
   const configuration = changes.map(change => ({
-    name: names[change.name],
+    name: change.name,
     value: Array.isArray(change.value) ? change.value.join(',') : change.value,
     origin: change.origin
   }))
 
-  sendData(config, application, host, 'app-client-configuration-change', {
-    configuration
-  })
+  const { reqType, payload } = createPayload('app-client-configuration-change', { configuration })
+
+  sendData(config, application, host, reqType, payload, updateRetryData)
 }
 
 module.exports = {

package/packages/dd-trace/src/telemetry/logs/index.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const dc = require('../../../../diagnostics_channel')
+const dc = require('dc-polyfill')
 const logCollector = require('./log-collector')
 const { sendData } = require('../send-data')
 
@@ -52,7 +52,7 @@ function stop () {
 function send (config, application, host) {
   if (!enabled) return
 
-  const logs = logCollector.drain()
+  const logs = { 'logs': logCollector.drain() }
   if (logs) {
     sendData(config, application, host, 'logs', logs)
   }

package/packages/dd-trace/src/telemetry/send-data.js

@@ -1,9 +1,12 @@
+
 const request = require('../exporters/common/request')
+const log = require('../log')
+let agentTelemetry = true
 
 function getHeaders (config, application, reqType) {
   const headers = {
     'content-type': 'application/json',
-    'dd-telemetry-api-version': 'v1',
+    'dd-telemetry-api-version': 'v2',
     'dd-telemetry-request-type': reqType,
     'dd-client-library-language': application.language_name,
     'dd-client-library-version': application.tracer_version
@@ -28,7 +31,7 @@ function getPayload (payload) {
   }
 }
 
-function sendData (config, application, host, reqType, payload = {}) {
+function sendData (config, application, host, reqType, payload = {}, cb = () => {}) {
   const {
     hostname,
     port,
@@ -44,7 +47,8 @@ function sendData (config, application, host, reqType, payload = {}) {
     headers: getHeaders(config, application, reqType)
   }
   const data = JSON.stringify({
-    api_version: 'v1',
+    api_version: 'v2',
+    naming_schema_version: config.spanAttributeSchema ? config.spanAttributeSchema : '',
     request_type: reqType,
     tracer_time: Math.floor(Date.now() / 1000),
     runtime_id: config.tags['runtime-id'],
@@ -54,8 +58,46 @@ function sendData (config, application, host, reqType, payload = {}) {
     host
   })
 
-  request(data, options, () => {
-    // ignore errors
+  request(data, options, (error) => {
+    if (error && process.env.DD_API_KEY && config.site) {
+      if (agentTelemetry) {
+        log.warn('Agent telemetry failed, started agentless telemetry')
+        agentTelemetry = false
+      }
+      // figure out which data center to send to
+      let backendUrl
+      const dataCenters = [
+        'datadoghq.com',
+        'us3.datadoghq.com',
+        'us5.datadoghq.com',
+        'ap1.datadoghq.com',
+        'eu1.datadoghq.com'
+      ]
+      if (config.site === 'datad0g.com') { // staging
+        backendUrl = 'https://all-http-intake.logs.datad0g.com/api/v2/apmtelemetry'
+      } else if (dataCenters.includes(config.site)) {
+        backendUrl = 'https://instrumentation-telemetry-intake.' + config.site + '/api/v2/apmtelemetry'
+      }
+      const backendHeader = { ...options.headers, 'DD-API-KEY': process.env.DD_API_KEY }
+      const backendOptions = {
+        ...options,
+        url: backendUrl,
+        headers: backendHeader
+      }
+      if (backendUrl) {
+        request(data, backendOptions, (error) => { log.error(error) })
+      } else {
+        log.error('Invalid Telemetry URL')
+      }
+    }
+
+    if (!error && !agentTelemetry) {
+      agentTelemetry = true
+      log.info('Started agent telemetry')
+    }
+
+    // call the callback function so that we can track the error and payload
+    cb(error, { payload, reqType })
   })
 }
 

package/packages/dd-trace/src/tracer.js

@@ -31,8 +31,10 @@ class DatadogTracer extends Tracer {
 
   // todo[piochelepiotr] These two methods are not related to the tracer, but to data streams monitoring.
   // They should be moved outside of the tracer in the future.
-  setCheckpoint (edgeTags) {
-    const ctx = this._dataStreamsProcessor.setCheckpoint(edgeTags, DataStreamsContext.getDataStreamsContext())
+  setCheckpoint (edgeTags, span, payloadSize = 0) {
+    const ctx = this._dataStreamsProcessor.setCheckpoint(
+      edgeTags, span, DataStreamsContext.getDataStreamsContext(), payloadSize
+    )
     DataStreamsContext.setDataStreamsContext(ctx)
     return ctx
   }
@@ -44,6 +46,10 @@ class DatadogTracer extends Tracer {
     return ctx
   }
 
+  setOffset (offsetData) {
+    return this._dataStreamsProcessor.setOffset(offsetData)
+  }
+
   trace (name, options, fn) {
     options = Object.assign({
       childOf: this.scope().active()

package/scripts/install_plugin_modules.js

@@ -80,7 +80,9 @@ async function assertVersions () {
 }
 
 async function assertInstrumentation (instrumentation, external) {
-  const versions = [].concat(instrumentation.versions || [])
+  const versions = process.env.PACKAGE_VERSION_RANGE ? [process.env.PACKAGE_VERSION_RANGE]
+    : [].concat(instrumentation.versions || [])
+
   for (const version of versions) {
     if (version) {
       await assertModules(instrumentation.name, semver.coerce(version).version, external)
@@ -130,8 +132,14 @@ async function assertPackage (name, version, dependency, external) {
   }
 
   if (!external) {
-    pkg.workspaces = {
-      nohoist: ['**/**']
+    if (name === 'aerospike') {
+      pkg.installConfig = {
+        'hoistingLimits': 'workspaces'
+      }
+    } else {
+      pkg.workspaces = {
+        nohoist: ['**/**']
+      }
     }
   }
   fs.writeFileSync(filename(name, version, 'package.json'), JSON.stringify(pkg, null, 2) + '\n')

package/packages/diagnostics_channel/index.js

@@ -1,3 +0,0 @@
-'use strict'
-
-module.exports = require('./src')