dd-trace 2.4.1 → 2.6.0

package/packages/dd-trace/src/exporters/common/request.js

@@ -0,0 +1,83 @@
+'use strict'
+
+const http = require('http')
+const https = require('https')
+const log = require('../../log')
+const docker = require('./docker')
+const { storage } = require('../../../../datadog-core')
+
+const httpAgent = new http.Agent({ keepAlive: true })
+const httpsAgent = new https.Agent({ keepAlive: true })
+const containerId = docker.id()
+
+function request (data, options, keepAlive, callback) {
+  if (!options.headers) {
+    options.headers = {}
+  }
+  const isSecure = options.protocol === 'https:'
+  const client = isSecure ? https : http
+  const dataArray = [].concat(data)
+  options.headers['Content-Length'] = byteLength(dataArray)
+
+  if (containerId) {
+    options.headers['Datadog-Container-ID'] = containerId
+  }
+
+  if (keepAlive) {
+    options.agent = isSecure ? httpsAgent : httpAgent
+  }
+
+  const firstRequest = retriableRequest(options, client, callback)
+  dataArray.forEach(buffer => firstRequest.write(buffer))
+
+  // The first request will be retried
+  const firstRequestErrorHandler = () => {
+    log.debug('Retrying request to the intake')
+    const retriedReq = retriableRequest(options, client, callback)
+    dataArray.forEach(buffer => retriedReq.write(buffer))
+    // The retried request will fail normally
+    retriedReq.on('error', e => callback(new Error(`Network error trying to reach the intake: ${e.message}`)))
+    retriedReq.end()
+  }
+
+  firstRequest.on('error', firstRequestErrorHandler)
+  firstRequest.end()
+
+  return firstRequest
+}
+
+function retriableRequest (options, client, callback) {
+  const store = storage.getStore()
+
+  storage.enterWith({ noop: true })
+
+  const timeout = options.timeout || 15000
+
+  const request = client.request(options, res => {
+    let responseData = ''
+
+    res.setTimeout(timeout)
+
+    res.on('data', chunk => { responseData += chunk })
+    res.on('end', () => {
+      if (res.statusCode >= 200 && res.statusCode <= 299) {
+        callback(null, responseData, res.statusCode)
+      } else {
+        const error = new Error(`Error from the endpoint: ${res.statusCode} ${http.STATUS_CODES[res.statusCode]}`)
+        error.status = res.statusCode
+
+        callback(error, null, res.statusCode)
+      }
+    })
+  })
+  request.setTimeout(timeout, request.abort)
+  storage.enterWith(store)
+
+  return request
+}
+
+function byteLength (data) {
+  return data.length > 0 ? data.reduce((prev, next) => prev + next.length, 0) : 0
+}
+
+module.exports = request

package/packages/dd-trace/src/exporters/common/writer.js

@@ -0,0 +1,36 @@
+'use strict'
+const log = require('../../log')
+
+class Writer {
+  constructor ({ url }) {
+    this._url = url
+  }
+
+  flush (done = () => {}) {
+    const count = this._encoder.count()
+
+    if (count > 0) {
+      const payload = this._encoder.makePayload()
+
+      this._sendPayload(payload, count, done)
+    } else {
+      done()
+    }
+  }
+
+  append (spans) {
+    log.debug(() => `Encoding trace: ${JSON.stringify(spans)}`)
+
+    this._encode(spans)
+  }
+
+  _encode (trace) {
+    this._encoder.encode(trace)
+  }
+
+  setUrl (url) {
+    this._url = url
+  }
+}
+
+module.exports = Writer

package/packages/dd-trace/src/format.js

@@ -22,7 +22,6 @@ const map = {
 function format (span) {
   const formatted = formatSpan(span)
 
-  extractError(formatted, span)
   extractRootTags(formatted, span)
   extractChunkTags(formatted, span)
   extractTags(formatted, span)
@@ -74,8 +73,8 @@ function extractTags (trace, span) {
         addTag({}, trace.metrics, tag, tags[tag] === undefined || tags[tag] ? 1 : 0)
         break
       case 'error':
-        if (tags[tag] && (context._name !== 'fs.operation')) {
-          trace.error = 1
+        if (context._name !== 'fs.operation') {
+          extractError(trace, tags[tag])
         }
         break
       case 'error.type':
@@ -84,6 +83,8 @@ function extractTags (trace, span) {
         // HACK: remove when implemented in the backend
         if (context._name !== 'fs.operation') {
           trace.error = 1
+        } else {
+          break
         }
       default: // eslint-disable-line no-fallthrough
         addTag(trace.meta, trace.metrics, tag, tags[tag])
@@ -122,8 +123,11 @@ function extractChunkTags (trace, span) {
   }
 }
 
-function extractError (trace, span) {
-  const error = span.context()._tags['error']
+function extractError (trace, error) {
+  if (!error) return
+
+  trace.error = 1
+
   if (isError(error)) {
     addTag(trace.meta, trace.metrics, 'error.msg', error.message)
     addTag(trace.meta, trace.metrics, 'error.type', error.name)

package/packages/dd-trace/src/instrumenter.js

@@ -7,6 +7,7 @@ const Loader = require('./loader')
 const { isTrue } = require('./util')
 const plugins = require('./plugins')
 const Plugin = require('./plugins/plugin')
+const telemetry = require('./telemetry')
 
 const disabledPlugins = process.env.DD_TRACE_DISABLED_PLUGINS
 
@@ -54,6 +55,7 @@ class Instrumenter {
 
     try {
       this._set(plugin, { name, config })
+      telemetry.updateIntegrations()
     } catch (e) {
       log.debug(`Could not find a plugin named "${name}".`)
     }
@@ -154,6 +156,7 @@ class Instrumenter {
 
     if (!instrumented) {
       this._instrumented.set(instrumentation, instrumented = new Set())
+      telemetry.updateIntegrations()
     }
 
     if (!instrumented.has(this._defaultExport(moduleExports))) {

package/packages/dd-trace/src/pkg.js

@@ -11,7 +11,14 @@ function findRoot () {
 
 function findPkg () {
   const cwd = findRoot()
-  const filePath = findUp('package.json', cwd)
+  const directory = path.resolve(cwd)
+  const res = path.parse(directory)
+
+  if (!res) return {}
+
+  const { root } = res
+
+  const filePath = findUp('package.json', root, directory)
 
   try {
     return JSON.parse(fs.readFileSync(filePath, 'utf8'))
@@ -20,18 +27,16 @@ function findPkg () {
   }
 }
 
-function findUp (name, cwd) {
-  let directory = path.resolve(cwd)
-  const { root } = path.parse(directory)
-
+function findUp (name, root, directory) {
   while (true) {
     const current = path.resolve(directory, name)
 
     if (fs.existsSync(current)) return current
+
     if (directory === root) return
 
     directory = path.dirname(directory)
   }
 }
 
-module.exports = findPkg()
+module.exports = Object.assign(findPkg(), { findRoot, findUp })

package/packages/dd-trace/src/plugins/util/test.js

@@ -1,4 +1,7 @@
 const path = require('path')
+const fs = require('fs')
+
+const ignore = require('ignore')
 
 const { getGitMetadata } = require('./git')
 const { getUserProviderGitMetadata } = require('./user-provided-git')
@@ -16,6 +19,10 @@ const {
 } = require('./tags')
 const id = require('../../id')
 
+const { SPAN_TYPE, RESOURCE_NAME, SAMPLING_PRIORITY } = require('../../../../../ext/tags')
+const { SAMPLING_RULE_DECISION } = require('../../constants')
+const { AUTO_KEEP } = require('../../../../../ext/priority')
+
 const TEST_FRAMEWORK = 'test.framework'
 const TEST_FRAMEWORK_VERSION = 'test.framework_version'
 const TEST_TYPE = 'test.type'
@@ -25,6 +32,7 @@ const TEST_STATUS = 'test.status'
 const TEST_PARAMETERS = 'test.parameters'
 const TEST_SKIP_REASON = 'test.skip_reason'
 const TEST_IS_RUM_ACTIVE = 'test.is_rum_active'
+const TEST_CODE_OWNERS = 'test.codeowners'
 
 const ERROR_TYPE = 'error.type'
 const ERROR_MESSAGE = 'error.msg'
@@ -35,6 +43,7 @@ const CI_APP_ORIGIN = 'ciapp-test'
 const JEST_TEST_RUNNER = 'test.jest.test_runner'
 
 module.exports = {
+  TEST_CODE_OWNERS,
   TEST_FRAMEWORK,
   TEST_FRAMEWORK_VERSION,
   JEST_TEST_RUNNER,
@@ -53,7 +62,10 @@ module.exports = {
   getTestParametersString,
   finishAllTraceSpans,
   getTestParentSpan,
-  getTestSuitePath
+  getTestSuitePath,
+  getCodeOwnersFileEntries,
+  getCodeOwnersForFilename,
+  getTestCommonTags
 }
 
 function getTestEnvironmentMetadata (testFramework, config) {
@@ -127,6 +139,20 @@ function getTestParentSpan (tracer) {
     'x-datadog-parent-id': '0000000000000000'
   })
 }
+
+function getTestCommonTags (name, suite, version) {
+  return {
+    [SPAN_TYPE]: 'test',
+    [TEST_TYPE]: 'test',
+    [SAMPLING_RULE_DECISION]: 1,
+    [SAMPLING_PRIORITY]: AUTO_KEEP,
+    [TEST_NAME]: name,
+    [TEST_SUITE]: suite,
+    [RESOURCE_NAME]: `${suite}.${name}`,
+    [TEST_FRAMEWORK_VERSION]: version
+  }
+}
+
 /**
  * We want to make sure that test suites are reported the same way for
  * every OS, so we replace `path.sep` by `/`
@@ -140,3 +166,55 @@ function getTestSuitePath (testSuiteAbsolutePath, sourceRoot) {
 
   return testSuitePath.replace(path.sep, '/')
 }
+
+const POSSIBLE_CODEOWNERS_LOCATIONS = [
+  'CODEOWNERS',
+  '.github/CODEOWNERS',
+  'docs/CODEOWNERS',
+  '.gitlab/CODEOWNERS'
+]
+
+function getCodeOwnersFileEntries (rootDir = process.cwd()) {
+  let codeOwnersContent
+
+  POSSIBLE_CODEOWNERS_LOCATIONS.forEach(location => {
+    try {
+      codeOwnersContent = fs.readFileSync(`${rootDir}/${location}`).toString()
+    } catch (e) {
+      // retry with next path
+    }
+  })
+  if (!codeOwnersContent) {
+    return null
+  }
+
+  const entries = []
+  const lines = codeOwnersContent.split('\n')
+
+  for (const line of lines) {
+    const [content] = line.split('#')
+    const trimmed = content.trim()
+    if (trimmed === '') continue
+    const [pattern, ...owners] = trimmed.split(/\s+/)
+    entries.push({ pattern, owners })
+  }
+  // Reverse because rules defined last take precedence
+  return entries.reverse()
+}
+
+function getCodeOwnersForFilename (filename, entries) {
+  if (!entries) {
+    return null
+  }
+  for (const entry of entries) {
+    try {
+      const isResponsible = ignore().add(entry.pattern).ignores(filename)
+      if (isResponsible) {
+        return JSON.stringify(entry.owners)
+      }
+    } catch (e) {
+      return null
+    }
+  }
+  return null
+}

package/packages/dd-trace/src/plugins/util/web.js

@@ -69,6 +69,17 @@ const web = {
     context.span = span
     context.res = res
 
+    if (!config.filter(req.url)) {
+      span.setTag(MANUAL_DROP, true)
+      span.context()._trace.isRecording = false
+    }
+
+    if (config.service) {
+      span.setTag(SERVICE_NAME, config.service)
+    }
+
+    analyticsSampler.sample(span, config.measured, true)
+
     return span
   },
   wrap (req) {
@@ -83,16 +94,6 @@ const web = {
   instrument (tracer, config, req, res, name, callback) {
     const span = this.startSpan(tracer, config, req, res, name)
 
-    if (!config.filter(req.url)) {
-      span.setTag(MANUAL_DROP, true)
-    }
-
-    if (config.service) {
-      span.setTag(SERVICE_NAME, config.service)
-    }
-
-    analyticsSampler.sample(span, config.measured, true)
-
     this.wrap(req)
 
     return callback && tracer.scope().activate(span, () => callback(span))

package/packages/dd-trace/src/profiling/exporters/agent.js

@@ -5,7 +5,7 @@ const { request } = require('http')
 const FormData = require('form-data')
 
 // TODO: avoid using dd-trace internals. Make this a separate module?
-const docker = require('../../exporters/agent/docker')
+const docker = require('../../exporters/common/docker')
 const version = require('../../../lib/version')
 
 const containerId = docker.id()

package/packages/dd-trace/src/profiling/profilers/cpu.js

@@ -3,7 +3,7 @@
 class NativeCpuProfiler {
   constructor (options = {}) {
     this.type = 'wall'
-    this._samplingInterval = options.samplingInterval || 10 * 1000
+    this._samplingInterval = options.samplingInterval || 1e6 / 99 // 99hz
     this._mapper = undefined
     this._pprof = undefined
   }

package/packages/dd-trace/src/proxy.js

@@ -10,6 +10,7 @@ const metrics = require('./metrics')
 const log = require('./log')
 const { isFalse } = require('./util')
 const { setStartupLogInstrumenter } = require('./startup-log')
+const telemetry = require('./telemetry')
 
 const noop = new NoopTracer()
 
@@ -63,6 +64,7 @@ class Tracer extends BaseTracer {
         this._instrumenter.enable(config)
         this._pluginManager.configure(config)
         setStartupLogInstrumenter(this._instrumenter)
+        telemetry.start(config, this._instrumenter, this._pluginManager)
       }
     } catch (e) {
       log.error(e)

package/packages/dd-trace/src/span_processor.js

@@ -32,7 +32,10 @@ class SpanProcessor {
         }
       }
 
-      this._exporter.export(formatted)
+      if (formatted.length !== 0 && trace.isRecording !== false) {
+        this._exporter.export(formatted)
+      }
+
       this._erase(trace, active)
     }
   }

package/packages/dd-trace/src/telemetry.js

@@ -0,0 +1,187 @@
+'use strict'
+
+const tracerVersion = require('../lib/version')
+const pkg = require('./pkg')
+const containerId = require('./exporters/common/docker').id()
+const requirePackageJson = require('./require-package-json')
+const path = require('path')
+const os = require('os')
+const request = require('./exporters/common/request')
+
+let config
+let instrumenter
+let pluginManager
+
+let seqId = 0
+let application
+let host
+let interval
+const sentIntegrations = new Set()
+
+function getIntegrations () {
+  const newIntegrations = []
+  for (const plugin of instrumenter._instrumented.keys()) {
+    if (sentIntegrations.has(plugin.name)) {
+      continue
+    }
+    newIntegrations.push({
+      name: plugin.name,
+      enabled: true,
+      auto_enabled: true
+    })
+    sentIntegrations.add(plugin.name)
+  }
+  for (const pluginName in pluginManager._pluginsByName) {
+    if (sentIntegrations.has(pluginName)) {
+      continue
+    }
+    newIntegrations.push({
+      name: pluginName,
+      enabled: pluginManager._pluginsByName[pluginName]._enabled,
+      auto_enabled: true
+    })
+    sentIntegrations.add(pluginName)
+  }
+  return newIntegrations
+}
+
+function getDependencies () {
+  const deps = []
+  const { dependencies } = pkg
+  if (!dependencies) {
+    return deps
+  }
+  const rootDir = pkg.findRoot()
+  for (const [name, version] of Object.entries(dependencies)) {
+    const dep = { name }
+    try {
+      dep.version = requirePackageJson(
+        path.join(rootDir, 'node_modules', name.replace('/', path.sep))
+      ).version
+    } catch (e) {
+      dep.version = version
+    }
+    deps.push(dep)
+  }
+  return deps
+}
+
+function flatten (input, result = [], prefix = [], traversedObjects = null) {
+  traversedObjects = traversedObjects || new WeakSet()
+  if (traversedObjects.has(input)) {
+    return
+  }
+  traversedObjects.add(input)
+  for (const [key, value] of Object.entries(input)) {
+    if (typeof value === 'object' && value !== null) {
+      flatten(value, result, [...prefix, key], traversedObjects)
+    } else {
+      result.push({ name: [...prefix, key].join('.'), value })
+    }
+  }
+  return result
+}
+
+function appStarted () {
+  return {
+    integrations: getIntegrations(),
+    dependencies: getDependencies(),
+    configuration: flatten(config),
+    additional_payload: []
+  }
+}
+
+function onBeforeExit () {
+  process.removeListener('beforeExit', onBeforeExit)
+  sendData('app-closing')
+}
+
+function createAppObject () {
+  return {
+    service_name: config.service,
+    env: config.env,
+    service_version: config.version,
+    tracer_version: tracerVersion,
+    language_name: 'nodejs',
+    language_version: process.versions.node
+  }
+}
+
+function createHostObject () {
+  return {
+    hostname: os.hostname(), // TODO is this enough?
+    container_id: containerId
+  }
+}
+
+function sendData (reqType, payload = {}) {
+  const {
+    hostname,
+    port
+  } = config
+  const options = {
+    hostname,
+    port,
+    method: 'POST',
+    path: '/telemetry/proxy/api/v2/apmtelemetry',
+    headers: {
+      'content-type': 'application/json',
+      'dd-telemetry-api-version': 'v1',
+      'dd-telemetry-request-type': reqType
+    }
+  }
+  const data = JSON.stringify({
+    api_version: 'v1',
+    request_type: reqType,
+    tracer_time: Math.floor(Date.now() / 1000),
+    runtime_id: config.tags['runtime-id'],
+    seq_id: ++seqId,
+    payload,
+    application,
+    host
+  })
+
+  request(data, options, true, () => {
+    // ignore errors
+  })
+}
+
+function start (aConfig, theInstrumenter, thePluginManager) {
+  if (!aConfig.telemetryEnabled) {
+    return
+  }
+  config = aConfig
+  instrumenter = theInstrumenter
+  pluginManager = thePluginManager
+  application = createAppObject()
+  host = createHostObject()
+  sendData('app-started', appStarted())
+  interval = setInterval(() => sendData('app-heartbeat'), 60000)
+  interval.unref()
+  process.on('beforeExit', onBeforeExit)
+}
+
+function stop () {
+  if (!config) {
+    return
+  }
+  clearInterval(interval)
+  process.removeListener('beforeExit', onBeforeExit)
+}
+
+function updateIntegrations () {
+  if (!config || !config.telemetryEnabled) {
+    return
+  }
+  const integrations = getIntegrations()
+  if (integrations.length === 0) {
+    return
+  }
+  sendData('app-integrations-change', { integrations })
+}
+
+module.exports = {
+  start,
+  stop,
+  updateIntegrations
+}

package/scripts/install_plugin_modules.js

@@ -186,6 +186,7 @@ const requirePackageJson = require('${requirePackageJsonPath}')
 
 module.exports = {
   get (id) { return require(id || '${name}') },
+  getPath (id) { return require.resolve(id || '${name}' ) },
   version () { return requirePackageJson('${name}', module).version }
 }
 `