dd-trace 2.14.0 → 2.16.0

package/packages/dd-trace/src/span_stats.js

@@ -0,0 +1,210 @@
+const os = require('os')
+const { version } = require('./pkg')
+const pkg = require('../../../package.json')
+
+const { LogCollapsingLowestDenseDDSketch } = require('@datadog/sketches-js')
+const { ORIGIN_KEY, TOP_LEVEL_KEY } = require('./constants')
+const {
+  MEASURED,
+  HTTP_STATUS_CODE
+} = require('../../../ext/tags')
+
+const { SpanStatsExporter } = require('./exporters/span-stats')
+
+const {
+  DEFAULT_SPAN_NAME,
+  DEFAULT_SERVICE_NAME
+} = require('./encode/tags-processors')
+
+class SpanAggStats {
+  constructor (aggKey) {
+    this.aggKey = aggKey
+    this.hits = 0
+    this.topLevelHits = 0
+    this.errors = 0
+    this.duration = 0
+    this.okDistribution = new LogCollapsingLowestDenseDDSketch(0.00775)
+    this.errorDistribution = new LogCollapsingLowestDenseDDSketch(0.00775)
+  }
+
+  record (span) {
+    const durationNs = span._duration * 1e6
+    this.hits++
+    this.duration += durationNs
+
+    if (span.metrics[TOP_LEVEL_KEY]) {
+      this.topLevelHits++
+    }
+
+    if (span.error) {
+      this.errors++
+      this.errorDistribution.accept(durationNs)
+    } else {
+      this.okDistribution.accept(durationNs)
+    }
+  }
+
+  toJSON () {
+    const {
+      name,
+      service,
+      resource,
+      type,
+      statusCode,
+      synthetics
+    } = this.aggKey
+
+    return {
+      Name: name,
+      Service: service,
+      Resource: resource,
+      Type: type,
+      HTTPStatusCode: statusCode,
+      Synthetics: synthetics,
+      Hits: this.hits,
+      TopLevelHits: this.topLevelHits,
+      Errors: this.errors,
+      Duration: this.duration,
+      OkSummary: this.okDistribution.toProto(),
+      ErrorSummary: this.errorDistribution.toProto()
+    }
+  }
+}
+
+class SpanAggKey {
+  constructor (span) {
+    this.name = span.name || DEFAULT_SPAN_NAME
+    this.service = span.service || DEFAULT_SERVICE_NAME
+    this.resource = span.resource || ''
+    this.type = span.type || ''
+    this.statusCode = span.meta[HTTP_STATUS_CODE] || 0
+    this.synthetics = span.meta[ORIGIN_KEY] === 'synthetics'
+  }
+
+  toString () {
+    return [
+      this.name,
+      this.service,
+      this.resource,
+      this.type,
+      this.statusCode,
+      this.synthetics
+    ].join(',')
+  }
+}
+
+class SpanBuckets extends Map {
+  forSpan (span) {
+    const aggKey = new SpanAggKey(span)
+    const key = aggKey.toString()
+
+    if (!this.has(key)) {
+      this.set(key, new SpanAggStats(aggKey))
+    }
+
+    return this.get(key)
+  }
+}
+
+class TimeBuckets extends Map {
+  forTime (time) {
+    if (!this.has(time)) {
+      this.set(time, new SpanBuckets())
+    }
+
+    return this.get(time)
+  }
+}
+
+class SpanStatsProcessor {
+  constructor ({
+    stats: {
+      enabled = false,
+      interval = 10
+    },
+    hostname,
+    port,
+    url,
+    env,
+    tags
+  } = {}) {
+    this.exporter = new SpanStatsExporter({
+      hostname,
+      port,
+      tags,
+      url
+    })
+    this.interval = interval
+    this.bucketSizeNs = interval * 1e9
+    this.buckets = new TimeBuckets()
+    this.hostname = os.hostname()
+    this.enabled = enabled
+    this.env = env
+    this.tags = tags || {}
+    this.sequence = 0
+
+    if (enabled) {
+      this.timer = setInterval(this.onInterval.bind(this), interval * 1e3)
+      this.timer.unref()
+    }
+  }
+
+  onInterval () {
+    const serialized = this._serializeBuckets()
+    if (!serialized) return
+
+    this.exporter.export({
+      Hostname: this.hostname,
+      Env: this.env,
+      Version: version,
+      Stats: serialized,
+      Lang: 'javascript',
+      TracerVersion: pkg.version,
+      RuntimeID: this.tags['runtime-id'],
+      Sequence: ++this.sequence
+    })
+  }
+
+  onSpanFinished (span) {
+    if (!this.enabled) return
+    if (!span.metrics[TOP_LEVEL_KEY] && !span.metrics[MEASURED]) return
+
+    const spanEndNs = span.startTime + span.duration
+    const bucketTime = spanEndNs - (spanEndNs % this.bucketSizeNs)
+
+    this.buckets.forTime(bucketTime)
+      .forSpan(span)
+      .record(span)
+  }
+
+  _serializeBuckets () {
+    const { bucketSizeNs } = this
+    const serializedBuckets = []
+
+    for (const [ timeNs, bucket ] of this.buckets.entries()) {
+      const bucketAggStats = []
+
+      for (const stats of bucket.values()) {
+        bucketAggStats.push(stats.toJSON())
+      }
+
+      serializedBuckets.push({
+        Start: timeNs,
+        Duration: bucketSizeNs,
+        Stats: bucketAggStats
+      })
+    }
+
+    this.buckets.clear()
+
+    return serializedBuckets
+  }
+}
+
+module.exports = {
+  SpanAggStats,
+  SpanAggKey,
+  SpanBuckets,
+  TimeBuckets,
+  SpanStatsProcessor
+}

package/packages/dd-trace/src/telemetry/dependencies.js

@@ -0,0 +1,83 @@
+'use strict'
+
+const path = require('path')
+const parse = require('module-details-from-path')
+const requirePackageJson = require('../require-package-json')
+const { sendData } = require('./send-data')
+const dc = require('diagnostics_channel')
+const { fileURLToPath } = require('url')
+
+const savedDependencies = []
+const detectedDependencyNames = new Set()
+const FILE_URI_START = `file://`
+const moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart')
+
+let immediate, config, application, host
+
+function waitAndSend (config, application, host) {
+  if (!immediate) {
+    immediate = setImmediate(() => {
+      immediate = null
+      if (savedDependencies.length > 0) {
+        const dependencies = savedDependencies.splice(0, 1000)
+        sendData(config, application, host, 'app-dependencies-loaded', { dependencies })
+        if (savedDependencies.length > 0) {
+          waitAndSend(config, application, host)
+        }
+      }
+    })
+    immediate.unref()
+  }
+}
+
+function onModuleLoad (data) {
+  if (data) {
+    let filename = data.filename
+    if (filename && filename.startsWith(FILE_URI_START)) {
+      try {
+        filename = fileURLToPath(filename)
+      } catch (e) {
+        // cannot transform url to path
+      }
+    }
+    const parseResult = filename && parse(filename)
+    const request = data.request || (parseResult && parseResult.name)
+    if (filename && request && isDependency(filename, request) && !detectedDependencyNames.has(request)) {
+      detectedDependencyNames.add(request)
+      if (parseResult) {
+        const { name, basedir } = parseResult
+        if (basedir) {
+          try {
+            const { version } = requirePackageJson(basedir, module)
+            savedDependencies.push({ name, version })
+            waitAndSend(config, application, host)
+          } catch (e) {
+            // can not read the package.json, do nothing
+          }
+        }
+      }
+    }
+  }
+}
+function start (_config, _application, _host) {
+  config = _config
+  application = _application
+  host = _host
+  moduleLoadStartChannel.subscribe(onModuleLoad)
+}
+
+function isDependency (filename, request) {
+  return request.indexOf(`.${path.sep}`) !== 0 && request.indexOf(path.sep) !== 0
+}
+
+function stop () {
+  config = null
+  application = null
+  host = null
+  detectedDependencyNames.clear()
+  savedDependencies.splice(0, savedDependencies.length)
+  if (moduleLoadStartChannel.hasSubscribers) {
+    moduleLoadStartChannel.unsubscribe(onModuleLoad)
+  }
+}
+module.exports = { start, stop }

package/packages/dd-trace/src/{telemetry.js → telemetry/index.js}

@@ -1,17 +1,14 @@
 'use strict'
 
-const tracerVersion = require('../../../package.json').version
-const pkg = require('./pkg')
-const containerId = require('./exporters/common/docker').id()
-const requirePackageJson = require('./require-package-json')
-const path = require('path')
+const tracerVersion = require('../../../../package.json').version
+const containerId = require('../exporters/common/docker').id()
 const os = require('os')
-const request = require('./exporters/common/request')
+const dependencies = require('./dependencies')
+const { sendData } = require('./send-data')
 
 let config
 let pluginManager
 
-let seqId = 0
 let application
 let host
 let interval
@@ -33,27 +30,6 @@ function getIntegrations () {
   return newIntegrations
 }
 
-function getDependencies () {
-  const deps = []
-  const { dependencies } = pkg
-  if (!dependencies) {
-    return deps
-  }
-  const rootDir = pkg.findRoot()
-  for (const [name, version] of Object.entries(dependencies)) {
-    const dep = { name }
-    try {
-      dep.version = requirePackageJson(
-        path.join(rootDir, 'node_modules', name.replace('/', path.sep))
-      ).version
-    } catch (e) {
-      dep.version = version
-    }
-    deps.push(dep)
-  }
-  return deps
-}
-
 function flatten (input, result = [], prefix = [], traversedObjects = null) {
   traversedObjects = traversedObjects || new WeakSet()
   if (traversedObjects.has(input)) {
@@ -73,7 +49,7 @@ function flatten (input, result = [], prefix = [], traversedObjects = null) {
 function appStarted () {
   return {
     integrations: getIntegrations(),
-    dependencies: getDependencies(),
+    dependencies: [],
     configuration: flatten(config),
     additional_payload: []
   }
@@ -81,7 +57,7 @@ function appStarted () {
 
 function onBeforeExit () {
   process.removeListener('beforeExit', onBeforeExit)
-  sendData('app-closing')
+  sendData(config, application, host, 'app-closing')
 }
 
 function createAppObject () {
@@ -102,38 +78,6 @@ function createHostObject () {
   }
 }
 
-function sendData (reqType, payload = {}) {
-  const {
-    hostname,
-    port
-  } = config
-  const options = {
-    hostname,
-    port,
-    method: 'POST',
-    path: '/telemetry/proxy/api/v2/apmtelemetry',
-    headers: {
-      'content-type': 'application/json',
-      'dd-telemetry-api-version': 'v1',
-      'dd-telemetry-request-type': reqType
-    }
-  }
-  const data = JSON.stringify({
-    api_version: 'v1',
-    request_type: reqType,
-    tracer_time: Math.floor(Date.now() / 1000),
-    runtime_id: config.tags['runtime-id'],
-    seq_id: ++seqId,
-    payload,
-    application,
-    host
-  })
-
-  request(data, options, () => {
-    // ignore errors
-  })
-}
-
 function start (aConfig, thePluginManager) {
   if (!aConfig.telemetryEnabled) {
     return
@@ -142,8 +86,9 @@ function start (aConfig, thePluginManager) {
   pluginManager = thePluginManager
   application = createAppObject()
   host = createHostObject()
-  sendData('app-started', appStarted())
-  interval = setInterval(() => sendData('app-heartbeat'), 60000)
+  dependencies.start(config, application, host)
+  sendData(config, application, host, 'app-started', appStarted())
+  interval = setInterval(() => sendData(config, application, host, 'app-heartbeat'), 60000)
   interval.unref()
   process.on('beforeExit', onBeforeExit)
 }
@@ -164,7 +109,7 @@ function updateIntegrations () {
   if (integrations.length === 0) {
     return
   }
-  sendData('app-integrations-change', { integrations })
+  sendData(config, application, host, 'app-integrations-change', { integrations })
 }
 
 module.exports = {

package/packages/dd-trace/src/telemetry/send-data.js

@@ -0,0 +1,35 @@
+const request = require('../exporters/common/request')
+let seqId = 0
+function sendData (config, application, host, reqType, payload = {}) {
+  const {
+    hostname,
+    port
+  } = config
+  const options = {
+    hostname,
+    port,
+    method: 'POST',
+    path: '/telemetry/proxy/api/v2/apmtelemetry',
+    headers: {
+      'content-type': 'application/json',
+      'dd-telemetry-api-version': 'v1',
+      'dd-telemetry-request-type': reqType
+    }
+  }
+  const data = JSON.stringify({
+    api_version: 'v1',
+    request_type: reqType,
+    tracer_time: Math.floor(Date.now() / 1000),
+    runtime_id: config.tags['runtime-id'],
+    seq_id: ++seqId,
+    payload,
+    application,
+    host
+  })
+
+  request(data, options, () => {
+    // ignore errors
+  })
+}
+
+module.exports = { sendData }

package/packages/dd-trace/src/plugins/util/redis.js

@@ -1,74 +0,0 @@
-'use strict'
-
-const analyticsSampler = require('../../analytics_sampler')
-const urlFilter = require('../util/urlfilter')
-const tx = require('./tx')
-
-const redis = {
-  // Ensure the configuration has the correct structure and defaults.
-  normalizeConfig (config) {
-    const filter = urlFilter.getFilter(config)
-
-    return Object.assign({}, config, {
-      filter
-    })
-  },
-
-  // Start a span for a Redis command.
-  instrument (tracer, config, db, command, args) {
-    const childOf = tracer.scope().active()
-    const span = tracer.startSpan('redis.command', {
-      childOf,
-      tags: {
-        'span.kind': 'client',
-        'resource.name': command,
-        'span.type': 'redis',
-        'db.type': 'redis',
-        'db.name': db || '0',
-        'redis.raw_command': formatCommand(command, args)
-      }
-    })
-
-    span.setTag('service.name', config.service || `${span.context()._tags['service.name']}-redis`)
-
-    analyticsSampler.sample(span, config.measured)
-
-    return span
-  }
-}
-
-function formatCommand (command, args) {
-  command = command.toUpperCase()
-
-  if (!args || command === 'AUTH') return command
-
-  for (let i = 0, l = args.length; i < l; i++) {
-    if (typeof args[i] === 'function') continue
-
-    command = `${command} ${formatArg(args[i])}`
-
-    if (command.length > 1000) return trim(command, 1000)
-  }
-
-  return command
-}
-
-function formatArg (arg) {
-  switch (typeof arg) {
-    case 'string':
-    case 'number':
-      return trim(String(arg), 100)
-    default:
-      return '?'
-  }
-}
-
-function trim (str, maxlen) {
-  if (str.length > maxlen) {
-    str = str.substr(0, maxlen - 3) + '...'
-  }
-
-  return str
-}
-
-module.exports = Object.assign({}, tx, redis)

package/packages/dd-trace/src/plugins/util/tx.js

@@ -1,75 +0,0 @@
-'use strict'
-
-const tx = {
-  // Set the outgoing host.
-  setHost (span, hostname, port) {
-    hostname && span.setTag('out.host', hostname)
-    port && span.setTag('out.port', port)
-  },
-
-  // Wrap a promise or a callback to also finish the span.
-  wrap (span, done) {
-    if (typeof done === 'function' || !done) {
-      return wrapCallback(span, done)
-    } else if (isPromise(done)) {
-      return wrapPromise(span, done)
-    } else if (done && done.length) {
-      return wrapArguments(span, done)
-    }
-  }
-}
-
-function wrapCallback (span, callback) {
-  const scope = span.tracer().scope()
-  const previous = scope.active()
-
-  return function (err) {
-    finish(span, err)
-
-    if (callback) {
-      return scope.activate(previous, () => callback.apply(this, arguments))
-    }
-  }
-}
-
-function wrapPromise (span, promise) {
-  promise.then(
-    () => finish(span),
-    err => finish(span, err)
-  )
-
-  return promise
-}
-
-function wrapArguments (span, args) {
-  const lastIndex = args.length - 1
-  const callback = args[lastIndex]
-
-  if (typeof callback === 'function') {
-    args[lastIndex] = wrapCallback(span, args[lastIndex])
-  }
-
-  return args
-}
-
-function finish (span, error) {
-  if (error) {
-    span.addTags({
-      'error.type': error.name,
-      'error.msg': error.message,
-      'error.stack': error.stack
-    })
-  }
-
-  span.finish()
-}
-
-function isPromise (obj) {
-  return isObject(obj) && typeof obj.then === 'function'
-}
-
-function isObject (obj) {
-  return typeof obj === 'object' && obj !== null
-}
-
-module.exports = tx