dd-trace 4.18.0 → 4.23.0

package/packages/dd-trace/src/telemetry/index.js

@@ -1,13 +1,11 @@
 'use strict'
-
 const tracerVersion = require('../../../../package.json').version
-const dc = require('../../../diagnostics_channel')
+const dc = require('dc-polyfill')
 const os = require('os')
 const dependencies = require('./dependencies')
 const { sendData } = require('./send-data')
-
+const { errors } = require('../startup-log')
 const { manager: metricsManager } = require('./metrics')
-const logs = require('./logs')
 
 const telemetryStartChannel = dc.channel('datadog:telemetry:start')
 const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
@@ -17,11 +15,53 @@ let pluginManager
 
 let application
 let host
-let interval
 let heartbeatTimeout
 let heartbeatInterval
+let extendedInterval
+let integrations
+let retryData = null
+const extendedHeartbeatPayload = {}
+
 const sentIntegrations = new Set()
 
+function getRetryData () {
+  return retryData
+}
+
+function updateRetryData (error, retryObj) {
+  if (error) {
+    if (retryObj.reqType === 'message-batch') {
+      const payload = retryObj.payload[0].payload
+      const reqType = retryObj.payload[0].request_type
+      retryData = { payload: payload, reqType: reqType }
+
+      // Since this payload failed twice it now gets save in to the extended heartbeat
+      const failedPayload = retryObj.payload[1].payload
+      const failedReqType = retryObj.payload[1].request_type
+
+      // save away the dependencies and integration request for extended heartbeat.
+      if (failedReqType === 'app-integrations-change') {
+        if (extendedHeartbeatPayload['integrations']) {
+          extendedHeartbeatPayload['integrations'].push(failedPayload)
+        } else {
+          extendedHeartbeatPayload['integrations'] = [failedPayload]
+        }
+      }
+      if (failedReqType === 'app-dependencies-loaded') {
+        if (extendedHeartbeatPayload['dependencies']) {
+          extendedHeartbeatPayload['dependencies'].push(failedPayload)
+        } else {
+          extendedHeartbeatPayload['dependencies'] = [failedPayload]
+        }
+      }
+    } else {
+      retryData = retryObj
+    }
+  } else {
+    retryData = null
+  }
+}
+
 function getIntegrations () {
   const newIntegrations = []
   for (const pluginName in pluginManager._pluginsByName) {
@@ -38,6 +78,23 @@ function getIntegrations () {
   return newIntegrations
 }
 
+function getProducts (config) {
+  const products = {
+    appsec: {
+      enabled: config.appsec.enabled
+    },
+    profiler: {
+      version: tracerVersion,
+      enabled: config.profiling.enabled
+    }
+  }
+  if (errors.profilingError) {
+    products.profiler.error = errors.profilingError
+    errors.profilingError = {}
+  }
+  return products
+}
+
 function flatten (input, result = [], prefix = [], traversedObjects = null) {
   traversedObjects = traversedObjects || new WeakSet()
   if (traversedObjects.has(input)) {
@@ -48,33 +105,49 @@ function flatten (input, result = [], prefix = [], traversedObjects = null) {
     if (typeof value === 'object' && value !== null) {
       flatten(value, result, [...prefix, key], traversedObjects)
     } else {
-      result.push({ name: [...prefix, key].join('.'), value })
+      // TODO: add correct origin value
+      result.push({ name: [...prefix, key].join('.'), value, origin: 'unknown' })
     }
   }
   return result
 }
 
-function appStarted () {
-  return {
-    integrations: getIntegrations(),
-    dependencies: [],
-    configuration: flatten(formatConfig(config)),
-    additional_payload: []
+function getInstallSignature (config) {
+  const { installSignature: sig } = config
+  if (sig && (sig.id || sig.time || sig.type)) {
+    return {
+      install_id: sig.id,
+      install_time: sig.time,
+      install_type: sig.type
+    }
   }
 }
 
-function formatConfig (config) {
-  // format peerServiceMapping from an object to a string map in order for
-  // telemetry intake to accept the configuration
-  config.peerServiceMapping = config.peerServiceMapping
-    ? Object.entries(config.peerServiceMapping).map(([key, value]) => `${key}:${value}`).join(',')
-    : ''
-  return config
+function appStarted (config) {
+  const app = {
+    products: getProducts(config),
+    configuration: flatten(config)
+  }
+  const installSignature = getInstallSignature(config)
+  if (installSignature) {
+    app.install_signature = installSignature
+  }
+  // TODO: add app.error with correct error codes
+  // if (errors.agentError) {
+  //   app.error = errors.agentError
+  //   errors.agentError = {}
+  // }
+  return app
 }
 
 function onBeforeExit () {
   process.removeListener('beforeExit', onBeforeExit)
-  sendData(config, application, host, 'app-closing')
+  const { reqType, payload } = createPayload('app-closing')
+  sendData(config, application, host, reqType, payload)
+  // we flush before shutting down. Only in CI Visibility
+  if (config.isCiVisibility) {
+    metricsManager.send(config, application, host)
+  }
 }
 
 function createAppObject (config) {
@@ -121,14 +194,52 @@ function getTelemetryData () {
   return { config, application, host, heartbeatInterval }
 }
 
+function createBatchPayload (payload) {
+  const batchPayload = []
+  payload.map(item => {
+    batchPayload.push({
+      request_type: item.reqType,
+      payload: item.payload
+    })
+  })
+
+  return batchPayload
+}
+
+function createPayload (currReqType, currPayload = {}) {
+  if (getRetryData()) {
+    const payload = { reqType: currReqType, payload: currPayload }
+    const batchPayload = createBatchPayload([payload, retryData])
+    return { 'reqType': 'message-batch', 'payload': batchPayload }
+  }
+
+  return { 'reqType': currReqType, 'payload': currPayload }
+}
+
 function heartbeat (config, application, host) {
   heartbeatTimeout = setTimeout(() => {
-    sendData(config, application, host, 'app-heartbeat')
+    metricsManager.send(config, application, host)
+
+    const { reqType, payload } = createPayload('app-heartbeat')
+    sendData(config, application, host, reqType, payload, updateRetryData)
     heartbeat(config, application, host)
   }, heartbeatInterval).unref()
   return heartbeatTimeout
 }
 
+function extendedHeartbeat (config) {
+  extendedInterval = setInterval(() => {
+    const appPayload = appStarted(config)
+    const payload = {
+      ...appPayload,
+      ...extendedHeartbeatPayload
+    }
+    sendData(config, application, host, 'app-extended-heartbeat', payload)
+    Object.keys(extendedHeartbeatPayload).forEach(key => delete extendedHeartbeatPayload[key])
+  }, 1000 * 60 * 60 * 24).unref()
+  return extendedInterval
+}
+
 function start (aConfig, thePluginManager) {
   if (!aConfig.telemetry.enabled) {
     return
@@ -138,19 +249,22 @@ function start (aConfig, thePluginManager) {
   application = createAppObject(config)
   host = createHostObject()
   heartbeatInterval = config.telemetry.heartbeatInterval
+  integrations = getIntegrations()
+
+  dependencies.start(config, application, host, getRetryData, updateRetryData)
 
-  dependencies.start(config, application, host)
-  logs.start(config)
+  sendData(config, application, host, 'app-started', appStarted(config))
+
+  if (integrations.length > 0) {
+    sendData(config, application, host, 'app-integrations-change',
+      { integrations }, updateRetryData)
+  }
 
-  sendData(config, application, host, 'app-started', appStarted())
   heartbeat(config, application, host)
-  interval = setInterval(() => {
-    metricsManager.send(config, application, host)
-    logs.send(config, application, host)
-  }, heartbeatInterval)
-  interval.unref()
-  process.on('beforeExit', onBeforeExit)
 
+  extendedHeartbeat(config)
+
+  process.on('beforeExit', onBeforeExit)
   telemetryStartChannel.publish(getTelemetryData())
 }
 
@@ -158,7 +272,7 @@ function stop () {
   if (!config) {
     return
   }
-  clearInterval(interval)
+  clearInterval(extendedInterval)
   clearTimeout(heartbeatTimeout)
   process.removeListener('beforeExit', onBeforeExit)
 
@@ -175,7 +289,10 @@ function updateIntegrations () {
   if (integrations.length === 0) {
     return
   }
-  sendData(config, application, host, 'app-integrations-change', { integrations })
+
+  const { reqType, payload } = createPayload('app-integrations-change', { integrations })
+
+  sendData(config, application, host, reqType, payload, updateRetryData)
 }
 
 function updateConfig (changes, config) {
@@ -191,18 +308,31 @@ function updateConfig (changes, config) {
   const names = {
     sampleRate: 'DD_TRACE_SAMPLE_RATE',
     logInjection: 'DD_LOG_INJECTION',
-    headerTags: 'DD_TRACE_HEADER_TAGS'
+    headerTags: 'DD_TRACE_HEADER_TAGS',
+    tags: 'DD_TAGS'
   }
 
-  const configuration = changes.map(change => ({
-    name: names[change.name],
-    value: Array.isArray(change.value) ? change.value.join(',') : change.value,
-    origin: change.origin
-  }))
+  const configuration = []
 
-  sendData(config, application, host, 'app-client-configuration-change', {
-    configuration
-  })
+  for (const change of changes) {
+    if (!names.hasOwnProperty(change.name)) continue
+
+    const name = names[change.name]
+    const { origin, value } = change
+    const entry = { name, origin, value }
+
+    if (Array.isArray(value)) {
+      entry.value = value.join(',')
+    } else if (name === 'DD_TAGS') {
+      entry.value = Object.entries(value).map(([key, value]) => `${key}:${value}`)
+    }
+
+    configuration.push(entry)
+  }
+
+  const { reqType, payload } = createPayload('app-client-configuration-change', { configuration })
+
+  sendData(config, application, host, reqType, payload, updateRetryData)
 }
 
 module.exports = {

package/packages/dd-trace/src/telemetry/logs/index.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const dc = require('../../../../diagnostics_channel')
+const dc = require('dc-polyfill')
 const logCollector = require('./log-collector')
 const { sendData } = require('../send-data')
 
@@ -52,7 +52,7 @@ function stop () {
 function send (config, application, host) {
   if (!enabled) return
 
-  const logs = logCollector.drain()
+  const logs = { 'logs': logCollector.drain() }
   if (logs) {
     sendData(config, application, host, 'logs', logs)
   }

package/packages/dd-trace/src/telemetry/send-data.js

@@ -1,9 +1,12 @@
+
 const request = require('../exporters/common/request')
+const log = require('../log')
+let agentTelemetry = true
 
 function getHeaders (config, application, reqType) {
   const headers = {
     'content-type': 'application/json',
-    'dd-telemetry-api-version': 'v1',
+    'dd-telemetry-api-version': 'v2',
     'dd-telemetry-request-type': reqType,
     'dd-client-library-language': application.language_name,
     'dd-client-library-version': application.tracer_version
@@ -28,7 +31,7 @@ function getPayload (payload) {
   }
 }
 
-function sendData (config, application, host, reqType, payload = {}) {
+function sendData (config, application, host, reqType, payload = {}, cb = () => {}) {
   const {
     hostname,
     port,
@@ -44,7 +47,8 @@ function sendData (config, application, host, reqType, payload = {}) {
     headers: getHeaders(config, application, reqType)
   }
   const data = JSON.stringify({
-    api_version: 'v1',
+    api_version: 'v2',
+    naming_schema_version: config.spanAttributeSchema ? config.spanAttributeSchema : '',
     request_type: reqType,
     tracer_time: Math.floor(Date.now() / 1000),
     runtime_id: config.tags['runtime-id'],
@@ -54,8 +58,46 @@ function sendData (config, application, host, reqType, payload = {}) {
     host
   })
 
-  request(data, options, () => {
-    // ignore errors
+  request(data, options, (error) => {
+    if (error && process.env.DD_API_KEY && config.site) {
+      if (agentTelemetry) {
+        log.warn('Agent telemetry failed, started agentless telemetry')
+        agentTelemetry = false
+      }
+      // figure out which data center to send to
+      let backendUrl
+      const dataCenters = [
+        'datadoghq.com',
+        'us3.datadoghq.com',
+        'us5.datadoghq.com',
+        'ap1.datadoghq.com',
+        'eu1.datadoghq.com'
+      ]
+      if (config.site === 'datad0g.com') { // staging
+        backendUrl = 'https://all-http-intake.logs.datad0g.com/api/v2/apmtelemetry'
+      } else if (dataCenters.includes(config.site)) {
+        backendUrl = 'https://instrumentation-telemetry-intake.' + config.site + '/api/v2/apmtelemetry'
+      }
+      const backendHeader = { ...options.headers, 'DD-API-KEY': process.env.DD_API_KEY }
+      const backendOptions = {
+        ...options,
+        url: backendUrl,
+        headers: backendHeader
+      }
+      if (backendUrl) {
+        request(data, backendOptions, (error) => { log.error(error) })
+      } else {
+        log.error('Invalid Telemetry URL')
+      }
+    }
+
+    if (!error && !agentTelemetry) {
+      agentTelemetry = true
+      log.info('Started agent telemetry')
+    }
+
+    // call the callback function so that we can track the error and payload
+    cb(error, { payload, reqType })
   })
 }
 

package/packages/dd-trace/src/tracer.js

@@ -31,8 +31,10 @@ class DatadogTracer extends Tracer {
 
   // todo[piochelepiotr] These two methods are not related to the tracer, but to data streams monitoring.
   // They should be moved outside of the tracer in the future.
-  setCheckpoint (edgeTags) {
-    const ctx = this._dataStreamsProcessor.setCheckpoint(edgeTags, DataStreamsContext.getDataStreamsContext())
+  setCheckpoint (edgeTags, span, payloadSize = 0) {
+    const ctx = this._dataStreamsProcessor.setCheckpoint(
+      edgeTags, span, DataStreamsContext.getDataStreamsContext(), payloadSize
+    )
     DataStreamsContext.setDataStreamsContext(ctx)
     return ctx
   }
@@ -44,6 +46,10 @@ class DatadogTracer extends Tracer {
     return ctx
   }
 
+  setOffset (offsetData) {
+    return this._dataStreamsProcessor.setOffset(offsetData)
+  }
+
   trace (name, options, fn) {
     options = Object.assign({
       childOf: this.scope().active()

package/scripts/install_plugin_modules.js

@@ -80,7 +80,9 @@ async function assertVersions () {
 }
 
 async function assertInstrumentation (instrumentation, external) {
-  const versions = [].concat(instrumentation.versions || [])
+  const versions = process.env.PACKAGE_VERSION_RANGE ? [process.env.PACKAGE_VERSION_RANGE]
+    : [].concat(instrumentation.versions || [])
+
   for (const version of versions) {
     if (version) {
       await assertModules(instrumentation.name, semver.coerce(version).version, external)
@@ -130,8 +132,14 @@ async function assertPackage (name, version, dependency, external) {
   }
 
   if (!external) {
-    pkg.workspaces = {
-      nohoist: ['**/**']
+    if (name === 'aerospike') {
+      pkg.installConfig = {
+        'hoistingLimits': 'workspaces'
+      }
+    } else {
+      pkg.workspaces = {
+        nohoist: ['**/**']
+      }
     }
   }
   fs.writeFileSync(filename(name, version, 'package.json'), JSON.stringify(pkg, null, 2) + '\n')

package/packages/diagnostics_channel/index.js

@@ -1,3 +0,0 @@
-'use strict'
-
-module.exports = require('./src')

package/packages/diagnostics_channel/src/index.js

@@ -1,121 +0,0 @@
-'use strict'
-
-const {
-  Channel,
-  channel
-} = require('diagnostics_channel') // eslint-disable-line n/no-restricted-require
-
-const [major, minor] = process.versions.node.split('.')
-const channels = new WeakSet()
-
-// Our own DC with a limited subset of functionality stable across Node versions.
-// TODO: Move the rest of the polyfill here.
-// TODO: Switch to using global subscribe/unsubscribe/hasSubscribers.
-const dc = { channel }
-
-// Prevent going to 0 subscribers to avoid bug in Node.
-// See https://github.com/nodejs/node/pull/47520
-if (major === '19' && minor === '9') {
-  dc.channel = function () {
-    const ch = channel.apply(this, arguments)
-
-    if (!channels.has(ch)) {
-      const subscribe = ch.subscribe
-      const unsubscribe = ch.unsubscribe
-
-      ch.subscribe = function () {
-        delete ch.subscribe
-        delete ch.unsubscribe
-
-        const result = subscribe.apply(this, arguments)
-
-        this.subscribe(() => {}) // Keep it active forever.
-
-        return result
-      }
-
-      if (ch.unsubscribe === Channel.prototype.unsubscribe) {
-        // Needed because another subscriber could have subscribed to something
-        // that we unsubscribe to before the library is loaded.
-        ch.unsubscribe = function () {
-          delete ch.subscribe
-          delete ch.unsubscribe
-
-          this.subscribe(() => {}) // Keep it active forever.
-
-          return unsubscribe.apply(this, arguments)
-        }
-      }
-
-      channels.add(ch)
-    }
-
-    return ch
-  }
-}
-
-if (!Channel.prototype.runStores) {
-  const ActiveChannelPrototype = getActiveChannelPrototype()
-
-  Channel.prototype.bindStore = ActiveChannelPrototype.bindStore = function (store, transform) {
-    if (!this._stores) {
-      this._stores = new Map()
-    }
-    this._stores.set(store, transform)
-  }
-
-  Channel.prototype.unbindStore = ActiveChannelPrototype.unbindStore = function (store) {
-    if (!this._stores) return
-    this._stores.delete(store)
-  }
-
-  Channel.prototype.runStores = ActiveChannelPrototype.runStores = function (data, fn, thisArg, ...args) {
-    if (!this._stores) return Reflect.apply(fn, thisArg, args)
-
-    let run = () => {
-      this.publish(data)
-      return Reflect.apply(fn, thisArg, args)
-    }
-
-    for (const entry of this._stores.entries()) {
-      const store = entry[0]
-      const transform = entry[1]
-      run = wrapStoreRun(store, data, run, transform)
-    }
-
-    return run()
-  }
-}
-
-function defaultTransform (data) {
-  return data
-}
-
-function wrapStoreRun (store, data, next, transform = defaultTransform) {
-  return () => {
-    let context
-    try {
-      context = transform(data)
-    } catch (err) {
-      process.nextTick(() => {
-        throw err
-      })
-      return next()
-    }
-
-    return store.run(context, next)
-  }
-}
-
-function getActiveChannelPrototype () {
-  const dummyChannel = channel('foo')
-  const listener = () => {}
-
-  dummyChannel.subscribe(listener)
-  const ActiveChannelPrototype = Object.getPrototypeOf(dummyChannel)
-  dummyChannel.unsubscribe(listener)
-
-  return ActiveChannelPrototype
-}
-
-module.exports = dc