dd-trace 2.12.1 → 3.2.0

package/packages/dd-trace/src/span_stats.js

@@ -0,0 +1,210 @@
+const os = require('os')
+const { version } = require('./pkg')
+const pkg = require('../../../package.json')
+
+const { LogCollapsingLowestDenseDDSketch } = require('@datadog/sketches-js')
+const { ORIGIN_KEY, TOP_LEVEL_KEY } = require('./constants')
+const {
+  MEASURED,
+  HTTP_STATUS_CODE
+} = require('../../../ext/tags')
+
+const { SpanStatsExporter } = require('./exporters/span-stats')
+
+const {
+  DEFAULT_SPAN_NAME,
+  DEFAULT_SERVICE_NAME
+} = require('./encode/tags-processors')
+
+class SpanAggStats {
+  constructor (aggKey) {
+    this.aggKey = aggKey
+    this.hits = 0
+    this.topLevelHits = 0
+    this.errors = 0
+    this.duration = 0
+    this.okDistribution = new LogCollapsingLowestDenseDDSketch(0.00775)
+    this.errorDistribution = new LogCollapsingLowestDenseDDSketch(0.00775)
+  }
+
+  record (span) {
+    const durationNs = span._duration * 1e6
+    this.hits++
+    this.duration += durationNs
+
+    if (span.metrics[TOP_LEVEL_KEY]) {
+      this.topLevelHits++
+    }
+
+    if (span.error) {
+      this.errors++
+      this.errorDistribution.accept(durationNs)
+    } else {
+      this.okDistribution.accept(durationNs)
+    }
+  }
+
+  toJSON () {
+    const {
+      name,
+      service,
+      resource,
+      type,
+      statusCode,
+      synthetics
+    } = this.aggKey
+
+    return {
+      Name: name,
+      Service: service,
+      Resource: resource,
+      Type: type,
+      HTTPStatusCode: statusCode,
+      Synthetics: synthetics,
+      Hits: this.hits,
+      TopLevelHits: this.topLevelHits,
+      Errors: this.errors,
+      Duration: this.duration,
+      OkSummary: this.okDistribution.toProto(),
+      ErrorSummary: this.errorDistribution.toProto()
+    }
+  }
+}
+
+class SpanAggKey {
+  constructor (span) {
+    this.name = span.name || DEFAULT_SPAN_NAME
+    this.service = span.service || DEFAULT_SERVICE_NAME
+    this.resource = span.resource || ''
+    this.type = span.type || ''
+    this.statusCode = span.meta[HTTP_STATUS_CODE] || 0
+    this.synthetics = span.meta[ORIGIN_KEY] === 'synthetics'
+  }
+
+  toString () {
+    return [
+      this.name,
+      this.service,
+      this.resource,
+      this.type,
+      this.statusCode,
+      this.synthetics
+    ].join(',')
+  }
+}
+
+class SpanBuckets extends Map {
+  forSpan (span) {
+    const aggKey = new SpanAggKey(span)
+    const key = aggKey.toString()
+
+    if (!this.has(key)) {
+      this.set(key, new SpanAggStats(aggKey))
+    }
+
+    return this.get(key)
+  }
+}
+
+class TimeBuckets extends Map {
+  forTime (time) {
+    if (!this.has(time)) {
+      this.set(time, new SpanBuckets())
+    }
+
+    return this.get(time)
+  }
+}
+
+class SpanStatsProcessor {
+  constructor ({
+    stats: {
+      enabled = false,
+      interval = 10
+    },
+    hostname,
+    port,
+    url,
+    env,
+    tags
+  } = {}) {
+    this.exporter = new SpanStatsExporter({
+      hostname,
+      port,
+      tags,
+      url
+    })
+    this.interval = interval
+    this.bucketSizeNs = interval * 1e9
+    this.buckets = new TimeBuckets()
+    this.hostname = os.hostname()
+    this.enabled = enabled
+    this.env = env
+    this.tags = tags || {}
+    this.sequence = 0
+
+    if (enabled) {
+      this.timer = setInterval(this.onInterval.bind(this), interval * 1e3)
+      this.timer.unref()
+    }
+  }
+
+  onInterval () {
+    const serialized = this._serializeBuckets()
+    if (!serialized) return
+
+    this.exporter.export({
+      Hostname: this.hostname,
+      Env: this.env,
+      Version: version,
+      Stats: serialized,
+      Lang: 'javascript',
+      TracerVersion: pkg.version,
+      RuntimeID: this.tags['runtime-id'],
+      Sequence: ++this.sequence
+    })
+  }
+
+  onSpanFinished (span) {
+    if (!this.enabled) return
+    if (!span.metrics[TOP_LEVEL_KEY] && !span.metrics[MEASURED]) return
+
+    const spanEndNs = span.startTime + span.duration
+    const bucketTime = spanEndNs - (spanEndNs % this.bucketSizeNs)
+
+    this.buckets.forTime(bucketTime)
+      .forSpan(span)
+      .record(span)
+  }
+
+  _serializeBuckets () {
+    const { bucketSizeNs } = this
+    const serializedBuckets = []
+
+    for (const [ timeNs, bucket ] of this.buckets.entries()) {
+      const bucketAggStats = []
+
+      for (const stats of bucket.values()) {
+        bucketAggStats.push(stats.toJSON())
+      }
+
+      serializedBuckets.push({
+        Start: timeNs,
+        Duration: bucketSizeNs,
+        Stats: bucketAggStats
+      })
+    }
+
+    this.buckets.clear()
+
+    return serializedBuckets
+  }
+}
+
+module.exports = {
+  SpanAggStats,
+  SpanAggKey,
+  SpanBuckets,
+  TimeBuckets,
+  SpanStatsProcessor
+}

package/packages/dd-trace/src/startup-log.js

@@ -5,38 +5,27 @@ const mainLogger = require('./log')
 const os = require('os')
 const { inspect } = require('util')
 const tracerVersion = require('../../../package.json').version
-const requirePackageJson = require('./require-package-json')
 
 const logger = Object.create(mainLogger)
 logger._enabled = true
 
 let config
-let instrumenter
+let pluginManager
 let samplingRules = []
-
 let alreadyRan = false
 
 function getIntegrationsAndAnalytics () {
   const integrations = new Set()
   const extras = {}
-  for (const plugin of instrumenter._instrumented.keys()) {
-    if (plugin.versions) {
-      try {
-        const version = requirePackageJson(plugin.name, module).version
-        integrations.add(`${plugin.name}@${version}`)
-      } catch (e) {
-        integrations.add(plugin.name)
-      }
-    } else {
-      integrations.add(plugin.name)
-    }
+  for (const pluginName in pluginManager._pluginsByName) {
+    integrations.add(pluginName)
   }
   extras.integrations_loaded = Array.from(integrations)
   return extras
 }
 
 function startupLog ({ agentError } = {}) {
-  if (!config || !instrumenter) {
+  if (!config || !pluginManager) {
     return
   }
 
@@ -106,7 +95,7 @@ function startupLog ({ agentError } = {}) {
   }
 
   config = undefined
-  instrumenter = undefined
+  pluginManager = undefined
   samplingRules = undefined
 }
 
@@ -114,8 +103,8 @@ function setStartupLogConfig (aConfig) {
   config = aConfig
 }
 
-function setStartupLogInstrumenter (theInstrumenter) {
-  instrumenter = theInstrumenter
+function setStartupLogPluginManager (thePluginManager) {
+  pluginManager = thePluginManager
 }
 
 function setSamplingRules (theRules) {
@@ -125,6 +114,6 @@ function setSamplingRules (theRules) {
 module.exports = {
   startupLog,
   setStartupLogConfig,
-  setStartupLogInstrumenter,
+  setStartupLogPluginManager,
   setSamplingRules
 }

package/packages/dd-trace/src/telemetry/dependencies.js

@@ -0,0 +1,83 @@
+'use strict'
+
+const path = require('path')
+const parse = require('module-details-from-path')
+const requirePackageJson = require('../require-package-json')
+const { sendData } = require('./send-data')
+const dc = require('diagnostics_channel')
+const { fileURLToPath } = require('url')
+
+const savedDependencies = []
+const detectedDependencyNames = new Set()
+const FILE_URI_START = `file://`
+const moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart')
+
+let immediate, config, application, host
+
+function waitAndSend (config, application, host) {
+  if (!immediate) {
+    immediate = setImmediate(() => {
+      immediate = null
+      if (savedDependencies.length > 0) {
+        const dependencies = savedDependencies.splice(0, 1000)
+        sendData(config, application, host, 'app-dependencies-loaded', { dependencies })
+        if (savedDependencies.length > 0) {
+          waitAndSend(config, application, host)
+        }
+      }
+    })
+    immediate.unref()
+  }
+}
+
+function onModuleLoad (data) {
+  if (data) {
+    let filename = data.filename
+    if (filename && filename.startsWith(FILE_URI_START)) {
+      try {
+        filename = fileURLToPath(filename)
+      } catch (e) {
+        // cannot transform url to path
+      }
+    }
+    const parseResult = filename && parse(filename)
+    const request = data.request || (parseResult && parseResult.name)
+    if (filename && request && isDependency(filename, request) && !detectedDependencyNames.has(request)) {
+      detectedDependencyNames.add(request)
+      if (parseResult) {
+        const { name, basedir } = parseResult
+        if (basedir) {
+          try {
+            const { version } = requirePackageJson(basedir, module)
+            savedDependencies.push({ name, version })
+            waitAndSend(config, application, host)
+          } catch (e) {
+            // can not read the package.json, do nothing
+          }
+        }
+      }
+    }
+  }
+}
+function start (_config, _application, _host) {
+  config = _config
+  application = _application
+  host = _host
+  moduleLoadStartChannel.subscribe(onModuleLoad)
+}
+
+function isDependency (filename, request) {
+  return request.indexOf(`.${path.sep}`) !== 0 && request.indexOf(path.sep) !== 0
+}
+
+function stop () {
+  config = null
+  application = null
+  host = null
+  detectedDependencyNames.clear()
+  savedDependencies.splice(0, savedDependencies.length)
+  if (moduleLoadStartChannel.hasSubscribers) {
+    moduleLoadStartChannel.unsubscribe(onModuleLoad)
+  }
+}
+module.exports = { start, stop }

package/packages/dd-trace/src/{telemetry.js → telemetry/index.js}

@@ -1,18 +1,14 @@
 'use strict'
 
-const tracerVersion = require('../../../package.json').version
-const pkg = require('./pkg')
-const containerId = require('./exporters/common/docker').id()
-const requirePackageJson = require('./require-package-json')
-const path = require('path')
+const tracerVersion = require('../../../../package.json').version
+const containerId = require('../exporters/common/docker').id()
 const os = require('os')
-const request = require('./exporters/common/request')
+const dependencies = require('./dependencies')
+const { sendData } = require('./send-data')
 
 let config
-let instrumenter
 let pluginManager
 
-let seqId = 0
 let application
 let host
 let interval
@@ -20,17 +16,6 @@ const sentIntegrations = new Set()
 
 function getIntegrations () {
   const newIntegrations = []
-  for (const plugin of instrumenter._instrumented.keys()) {
-    if (sentIntegrations.has(plugin.name)) {
-      continue
-    }
-    newIntegrations.push({
-      name: plugin.name,
-      enabled: true,
-      auto_enabled: true
-    })
-    sentIntegrations.add(plugin.name)
-  }
   for (const pluginName in pluginManager._pluginsByName) {
     if (sentIntegrations.has(pluginName)) {
       continue
@@ -45,27 +30,6 @@ function getIntegrations () {
   return newIntegrations
 }
 
-function getDependencies () {
-  const deps = []
-  const { dependencies } = pkg
-  if (!dependencies) {
-    return deps
-  }
-  const rootDir = pkg.findRoot()
-  for (const [name, version] of Object.entries(dependencies)) {
-    const dep = { name }
-    try {
-      dep.version = requirePackageJson(
-        path.join(rootDir, 'node_modules', name.replace('/', path.sep))
-      ).version
-    } catch (e) {
-      dep.version = version
-    }
-    deps.push(dep)
-  }
-  return deps
-}
-
 function flatten (input, result = [], prefix = [], traversedObjects = null) {
   traversedObjects = traversedObjects || new WeakSet()
   if (traversedObjects.has(input)) {
@@ -85,7 +49,7 @@ function flatten (input, result = [], prefix = [], traversedObjects = null) {
 function appStarted () {
   return {
     integrations: getIntegrations(),
-    dependencies: getDependencies(),
+    dependencies: [],
     configuration: flatten(config),
     additional_payload: []
   }
@@ -93,7 +57,7 @@ function appStarted () {
 
 function onBeforeExit () {
   process.removeListener('beforeExit', onBeforeExit)
-  sendData('app-closing')
+  sendData(config, application, host, 'app-closing')
 }
 
 function createAppObject () {
@@ -114,49 +78,17 @@ function createHostObject () {
   }
 }
 
-function sendData (reqType, payload = {}) {
-  const {
-    hostname,
-    port
-  } = config
-  const options = {
-    hostname,
-    port,
-    method: 'POST',
-    path: '/telemetry/proxy/api/v2/apmtelemetry',
-    headers: {
-      'content-type': 'application/json',
-      'dd-telemetry-api-version': 'v1',
-      'dd-telemetry-request-type': reqType
-    }
-  }
-  const data = JSON.stringify({
-    api_version: 'v1',
-    request_type: reqType,
-    tracer_time: Math.floor(Date.now() / 1000),
-    runtime_id: config.tags['runtime-id'],
-    seq_id: ++seqId,
-    payload,
-    application,
-    host
-  })
-
-  request(data, options, true, () => {
-    // ignore errors
-  })
-}
-
-function start (aConfig, theInstrumenter, thePluginManager) {
+function start (aConfig, thePluginManager) {
   if (!aConfig.telemetryEnabled) {
     return
   }
   config = aConfig
-  instrumenter = theInstrumenter
   pluginManager = thePluginManager
   application = createAppObject()
   host = createHostObject()
-  sendData('app-started', appStarted())
-  interval = setInterval(() => sendData('app-heartbeat'), 60000)
+  dependencies.start(config, application, host)
+  sendData(config, application, host, 'app-started', appStarted())
+  interval = setInterval(() => sendData(config, application, host, 'app-heartbeat'), 60000)
   interval.unref()
   process.on('beforeExit', onBeforeExit)
 }
@@ -177,7 +109,7 @@ function updateIntegrations () {
   if (integrations.length === 0) {
     return
   }
-  sendData('app-integrations-change', { integrations })
+  sendData(config, application, host, 'app-integrations-change', { integrations })
 }
 
 module.exports = {

package/packages/dd-trace/src/telemetry/send-data.js

@@ -0,0 +1,35 @@
+const request = require('../exporters/common/request')
+let seqId = 0
+function sendData (config, application, host, reqType, payload = {}) {
+  const {
+    hostname,
+    port
+  } = config
+  const options = {
+    hostname,
+    port,
+    method: 'POST',
+    path: '/telemetry/proxy/api/v2/apmtelemetry',
+    headers: {
+      'content-type': 'application/json',
+      'dd-telemetry-api-version': 'v1',
+      'dd-telemetry-request-type': reqType
+    }
+  }
+  const data = JSON.stringify({
+    api_version: 'v1',
+    request_type: reqType,
+    tracer_time: Math.floor(Date.now() / 1000),
+    runtime_id: config.tags['runtime-id'],
+    seq_id: ++seqId,
+    payload,
+    application,
+    host
+  })
+
+  request(data, options, () => {
+    // ignore errors
+  })
+}
+
+module.exports = { sendData }

package/packages/dd-trace/src/tracer.js

@@ -107,10 +107,6 @@ class DatadogTracer extends Tracer {
     return this._scope
   }
 
-  currentSpan () {
-    return this.scope().active()
-  }
-
   getRumData () {
     if (!this._enableGetRumData) {
       return ''

package/scripts/install_plugin_modules.js

@@ -7,8 +7,6 @@ const semver = require('semver')
 const proxyquire = require('proxyquire')
 const exec = require('./helpers/exec')
 const childProcess = require('child_process')
-const plugins = require('../packages/dd-trace/src/plugins')
-const Plugin = require('../packages/dd-trace/src/plugins/plugin')
 const externals = require('../packages/dd-trace/test/plugins/externals')
 
 const requirePackageJsonPath = require.resolve('../packages/dd-trace/src/require-package-json')
@@ -16,6 +14,9 @@ const requirePackageJsonPath = require.resolve('../packages/dd-trace/src/require
 const workspaces = new Set()
 const versionLists = {}
 const deps = {}
+const names = []
+const filter = process.env.hasOwnProperty('PLUGINS') && process.env.PLUGINS.split('|')
+
 Object.keys(externals).forEach(external => externals[external].forEach(thing => {
   if (thing.dep) {
     if (!deps[external]) {
@@ -29,7 +30,10 @@ fs.readdirSync(path.join(__dirname, '../packages/datadog-instrumentations/src'))
   .filter(file => file.endsWith('js'))
   .forEach(file => {
     file = file.replace('.js', '')
-    plugins[file] = { name: file, prototype: Object.create(Plugin.prototype) }
+
+    if (!filter || filter.includes(file)) {
+      names.push(file)
+    }
   })
 
 run()
@@ -42,32 +46,19 @@ async function run () {
 }
 
 async function assertVersions () {
-  let filter = []
-  let names = Object.keys(plugins)
-
-  if (process.env.hasOwnProperty('PLUGINS')) {
-    filter = process.env.PLUGINS.split('|')
-    names = names.filter(name => ~filter.indexOf(name))
-  }
-
   const internals = names
     .map(key => {
-      const plugin = plugins[key]
-      if (plugin.prototype instanceof Plugin) {
-        const instrumentations = []
-        const name = plugin.name
-
-        try {
-          loadInstFile(`${name}/server.js`, instrumentations)
-          loadInstFile(`${name}/client.js`, instrumentations)
-        } catch (e) {
-          loadInstFile(`${name}.js`, instrumentations)
-        }
-
-        return instrumentations
-      } else {
-        return plugin
+      const instrumentations = []
+      const name = key
+
+      try {
+        loadInstFile(`${name}/server.js`, instrumentations)
+        loadInstFile(`${name}/client.js`, instrumentations)
+      } catch (e) {
+        loadInstFile(`${name}.js`, instrumentations)
       }
+
+      return instrumentations
     })
     .reduce((prev, next) => prev.concat(next), [])
 

package/ci/jest/env.js

@@ -1,39 +0,0 @@
-/* eslint-disable no-console */
-
-const tracer = require('../../packages/dd-trace')
-const { ORIGIN_KEY } = require('../../packages/dd-trace/src/constants')
-
-const options = {
-  startupLogs: false,
-  tags: {
-    [ORIGIN_KEY]: 'ciapp-test'
-  }
-}
-
-const isAgentlessEnabled = process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED &&
-  process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED !== 'false' &&
-  process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED !== '0'
-
-if (isAgentlessEnabled) {
-  if (process.env.DATADOG_API_KEY || process.env.DD_API_KEY) {
-    tracer.init({
-      ...options,
-      experimental: {
-        exporter: 'datadog'
-      }
-    })
-  } else {
-    console.error(`DD_CIVISIBILITY_AGENTLESS_ENABLED is set, \
-but neither DD_API_KEY nor DATADOG_API_KEY are set in your environment, \
-so dd-trace will not be initialized.`)
-  }
-} else {
-  tracer.init({
-    ...options,
-    flushInterval: 400000
-  })
-}
-
-tracer.use('fs', false)
-
-module.exports = tracer

package/cypress/plugin.js

@@ -1,5 +0,0 @@
-require('..').init({
-  startupLogs: false
-})
-
-module.exports = require('../packages/datadog-plugin-cypress/src/plugin')

package/cypress/support.js

@@ -1 +0,0 @@
-require('../packages/datadog-plugin-cypress/src/support')