dd-trace 2.12.1 → 2.14.0

package/packages/datadog-plugin-fs/src/index.js

@@ -1,10 +1,18 @@
 'use strict'
 
+// The `fs` plugin is an old style plugin that has not been updated for the new
+// plugin system and was hacked in for backward compatibility with 2.x.
+
 const { storage } = require('../../datadog-core')
+const { channel } = require('../../datadog-instrumentations/src/helpers/instrument')
+const shimmer = require('../../datadog-shimmer')
+const Plugin = require('../../dd-trace/src/plugins/plugin')
 
 let kDirReadPromisified
 let kDirClosePromisified
 let kHandle
+let fsConfig
+let fsInstance
 
 const ddFhSym = Symbol('ddFileHandle')
 
@@ -144,8 +152,8 @@ function createWrapDirAsyncIterator (config, tracer, instrumenter) {
           }
         }
       }
-      instrumenter.wrap(this, kDirReadPromisified, createWrapDirRead(config, tracer))
-      instrumenter.wrap(this, kDirClosePromisified, createWrapKDirClose(config, tracer, instrumenter))
+      shimmer.wrap(this, kDirReadPromisified, createWrapDirRead(config, tracer))
+      shimmer.wrap(this, kDirClosePromisified, createWrapKDirClose(config, tracer, instrumenter))
       return asyncIterator.apply(this, arguments)
     }
   }
@@ -158,8 +166,8 @@ function createWrapKDirClose (config, tracer, instrumenter) {
       return tracer.trace('fs.operation', { tags, orphanable }, (span) => {
         const p = kDirClose.apply(this, arguments)
         const unwrapBoth = () => {
-          instrumenter.unwrap(this, kDirReadPromisified)
-          instrumenter.unwrap(this, kDirClosePromisified)
+          shimmer.unwrap(this, kDirReadPromisified)
+          shimmer.unwrap(this, kDirClosePromisified)
         }
         p.then(unwrapBoth, unwrapBoth)
         return p
@@ -339,7 +347,7 @@ function makeFSTags (resourceName, path, options, config, tracer) {
     'component': 'fs',
     'span.kind': 'internal',
     'resource.name': resourceName,
-    'service.name': config.service || tracer._service
+    'service.name': fsConfig.service || tracer._service
   }
 
   switch (typeof path) {
@@ -399,9 +407,9 @@ function patchClassicFunctions (fs, tracer, config) {
     if (tagMakerName in tagMakers) {
       const tagMaker = tagMakers[tagMakerName]
       if (name.endsWith('Sync')) {
-        this.wrap(fs, name, createWrap(tracer, config, name, tagMaker))
+        shimmer.wrap(fs, name, createWrap(tracer, config, name, tagMaker))
       } else {
-        this.wrap(fs, name, createWrapCb(tracer, config, name, tagMaker))
+        shimmer.wrap(fs, name, createWrapCb(tracer, config, name, tagMaker))
       }
       if (name in promisifiable) {
         copySymbols(original, fs[name])
@@ -424,8 +432,6 @@ function patchFileHandle (fs, tracer, config) {
         tagMaker = createFDTags
       }
 
-      const instrumenter = this
-
       const desc = Reflect.getOwnPropertyDescriptor(fileHandlePrototype, kHandle)
       if (!desc || !desc.get) {
         Reflect.defineProperty(fileHandlePrototype, kHandle, {
@@ -434,13 +440,13 @@ function patchFileHandle (fs, tracer, config) {
           },
           set (h) {
             this[ddFhSym] = h
-            instrumenter.wrap(this, 'close', createWrap(tracer, config, 'filehandle.close', tagMakers.close))
+            shimmer.wrap(this, 'close', createWrap(tracer, config, 'filehandle.close', tagMakers.close))
           },
           configurable: true
         })
       }
 
-      this.wrap(fileHandlePrototype, name, createWrap(tracer, config, 'filehandle.' + name, tagMaker))
+      shimmer.wrap(fileHandlePrototype, name, createWrap(tracer, config, 'filehandle.' + name, tagMaker))
     }
   })
 }
@@ -449,17 +455,17 @@ function patchPromiseFunctions (fs, tracer, config) {
   for (const name in fs.promises) {
     if (name in tagMakers) {
       const tagMaker = tagMakers[name]
-      this.wrap(fs.promises, name, createWrap(tracer, config, 'promises.' + name, tagMaker))
+      shimmer.wrap(fs.promises, name, createWrap(tracer, config, 'promises.' + name, tagMaker))
     }
   }
 }
 
 function patchDirFunctions (fs, tracer, config) {
-  this.wrap(fs.Dir.prototype, 'close', createWrapDirClose(config, tracer))
-  this.wrap(fs.Dir.prototype, 'closeSync', createWrapDirClose(config, tracer, true))
-  this.wrap(fs.Dir.prototype, 'read', createWrapDirRead(config, tracer))
-  this.wrap(fs.Dir.prototype, 'readSync', createWrapDirRead(config, tracer, true))
-  this.wrap(fs.Dir.prototype, Symbol.asyncIterator, createWrapDirAsyncIterator(config, tracer, this))
+  shimmer.wrap(fs.Dir.prototype, 'close', createWrapDirClose(config, tracer))
+  shimmer.wrap(fs.Dir.prototype, 'closeSync', createWrapDirClose(config, tracer, true))
+  shimmer.wrap(fs.Dir.prototype, 'read', createWrapDirRead(config, tracer))
+  shimmer.wrap(fs.Dir.prototype, 'readSync', createWrapDirRead(config, tracer, true))
+  shimmer.wrap(fs.Dir.prototype, Symbol.asyncIterator, createWrapDirAsyncIterator(config, tracer, this))
 }
 
 function unpatchClassicFunctions (fs) {
@@ -467,7 +473,7 @@ function unpatchClassicFunctions (fs) {
     if (!fs[name]) continue
     const tagMakerName = name.endsWith('Sync') ? name.substr(0, name.length - 4) : name
     if (tagMakerName in tagMakers) {
-      this.unwrap(fs, name)
+      shimmer.unwrap(fs, name)
     }
   }
 }
@@ -478,7 +484,7 @@ function unpatchFileHandle (fs) {
       if (typeof name !== 'string' || name === 'constructor' || name === 'fd' || name === 'getAsyncId') {
         continue
       }
-      this.unwrap(fileHandlePrototype, name)
+      shimmer.unwrap(fileHandlePrototype, name)
     }
     delete fileHandlePrototype[kHandle]
   })
@@ -487,22 +493,46 @@ function unpatchFileHandle (fs) {
 function unpatchPromiseFunctions (fs) {
   for (const name in fs.promises) {
     if (name in tagMakers) {
-      this.unwrap(fs.promises, name)
+      shimmer.unwrap(fs.promises, name)
     }
   }
 }
 
 function unpatchDirFunctions (fs) {
-  this.unwrap(fs.Dir.prototype, 'close')
-  this.unwrap(fs.Dir.prototype, 'closeSync')
-  this.unwrap(fs.Dir.prototype, 'read')
-  this.unwrap(fs.Dir.prototype, 'readSync')
-  this.unwrap(fs.Dir.prototype, Symbol.asyncIterator)
+  shimmer.unwrap(fs.Dir.prototype, 'close')
+  shimmer.unwrap(fs.Dir.prototype, 'closeSync')
+  shimmer.unwrap(fs.Dir.prototype, 'read')
+  shimmer.unwrap(fs.Dir.prototype, 'readSync')
+  shimmer.unwrap(fs.Dir.prototype, Symbol.asyncIterator)
 }
 
-module.exports = {
-  name: 'fs',
-  patch (fs, tracer, config) {
+const hookChannel = channel('apm:fs:hook')
+
+hookChannel.subscribe(fs => {
+  fsInstance = fs
+})
+
+class FsPlugin extends Plugin {
+  static get name () {
+    return 'fs'
+  }
+
+  configure (config) {
+    fsConfig = config
+
+    super.configure(config)
+
+    this._unpatch()
+
+    if (this._enabled) {
+      this._patch()
+    }
+  }
+
+  _patch () {
+    const fs = fsInstance
+    const tracer = this.tracer
+    const config = this.config
     const realpathNative = fs.realpath.native
     const realpathSyncNative = fs.realpathSync.native
     patchClassicFunctions.call(this, fs, tracer, config)
@@ -513,18 +543,20 @@ module.exports = {
     if (fs.Dir) {
       patchDirFunctions.call(this, fs, tracer, config)
     }
-    this.wrap(fs, 'createReadStream', createWrapCreateReadStream(config, tracer))
-    this.wrap(fs, 'createWriteStream', createWrapCreateWriteStream(config, tracer))
-    this.wrap(fs, 'existsSync', createWrap(tracer, config, 'existsSync', createPathTags))
-    this.wrap(fs, 'exists', createWrapExists(config, tracer))
+    shimmer.wrap(fs, 'createReadStream', createWrapCreateReadStream(config, tracer))
+    shimmer.wrap(fs, 'createWriteStream', createWrapCreateWriteStream(config, tracer))
+    shimmer.wrap(fs, 'existsSync', createWrap(tracer, config, 'existsSync', createPathTags))
+    shimmer.wrap(fs, 'exists', createWrapExists(config, tracer))
     if (realpathNative) {
       fs.realpath.native = createWrapCb(tracer, config, 'realpath.native', createPathTags)(realpathNative)
     }
     if (realpathSyncNative) {
       fs.realpathSync.native = createWrap(tracer, config, 'realpath.native', createPathTags)(realpathSyncNative)
     }
-  },
-  unpatch (fs) {
+  }
+
+  _unpatch () {
+    const fs = fsInstance
     unpatchClassicFunctions.call(this, fs)
     if (fs.promises) {
       unpatchFileHandle.call(this, fs)
@@ -533,13 +565,15 @@ module.exports = {
     if (fs.Dir) {
       unpatchDirFunctions.call(this, fs)
     }
-    this.unwrap(fs, 'createReadStream')
-    this.unwrap(fs, 'createWriteStream')
-    this.unwrap(fs, 'existsSync')
-    this.unwrap(fs, 'exists')
+    shimmer.unwrap(fs, 'createReadStream')
+    shimmer.unwrap(fs, 'createWriteStream')
+    shimmer.unwrap(fs, 'existsSync')
+    shimmer.unwrap(fs, 'exists')
   }
 }
 
+module.exports = FsPlugin
+
 /** TODO fs functions:
 
 unwatchFile

package/packages/datadog-plugin-jest/src/index.js

@@ -15,6 +15,9 @@ const {
   TEST_CODE_OWNERS
 } = require('../../dd-trace/src/plugins/util/test')
 
+// https://github.com/facebook/jest/blob/d6ad15b0f88a05816c2fe034dd6900d28315d570/packages/jest-worker/src/types.ts#L38
+const CHILD_MESSAGE_END = 2
+
 function getTestSpanMetadata (tracer, test) {
   const childOf = getTestParentSpan(tracer)
 
@@ -38,9 +41,31 @@ class JestPlugin extends Plugin {
   constructor (...args) {
     super(...args)
 
+    // Used to handle the end of a jest worker to be able to flush
+    const handler = ([message]) => {
+      if (message === CHILD_MESSAGE_END) {
+        this.tracer._exporter._writer.flush(() => {
+          // eslint-disable-next-line
+          // https://github.com/facebook/jest/blob/24ed3b5ecb419c023ee6fdbc838f07cc028fc007/packages/jest-worker/src/workers/processChild.ts#L118-L133
+          // Only after the flush is done we clean up open handles
+          // so the worker process can hopefully exit gracefully
+          process.removeListener('message', handler)
+        })
+      }
+    }
+    process.on('message', handler)
+
     this.testEnvironmentMetadata = getTestEnvironmentMetadata('jest', this.config)
     this.codeOwnersEntries = getCodeOwnersFileEntries()
 
+    this.addSub('ci:jest:test:code-coverage', (coverageFiles) => {
+      if (!this.config.isAgentlessEnabled || !this.config.isIntelligentTestRunnerEnabled) {
+        return
+      }
+      const testSpan = storage.getStore().span
+      this.tracer._exporter.exportCoverage({ testSpan, coverageFiles })
+    })
+
     this.addSub('ci:jest:test:start', (test) => {
       const store = storage.getStore()
       const span = this.startTestSpan(test)
@@ -55,10 +80,6 @@ class JestPlugin extends Plugin {
       finishAllTraceSpans(span)
     })
 
-    this.addSub('ci:jest:test-suite:finish', () => {
-      this.tracer._exporter._writer.flush()
-    })
-
     this.addSub('ci:jest:test:err', (error) => {
       if (error) {
         const span = storage.getStore().span

package/packages/datadog-plugin-mocha/src/index.js

@@ -161,12 +161,12 @@ class MochaPlugin extends Plugin {
     const testSuiteSpan = this._testSuites.get(test.parent)
 
     if (testSuiteSpan) {
-      const testSuiteId = testSuiteSpan.context()._spanId.toString('hex')
+      const testSuiteId = testSuiteSpan.context()._spanId.toString(16)
       testSuiteTags[TEST_SUITE_ID] = testSuiteId
     }
 
     if (this.testSessionSpan) {
-      const testSessionId = this.testSessionSpan.context()._traceId.toString('hex')
+      const testSessionId = this.testSessionSpan.context()._traceId.toString(16)
       testSuiteTags[TEST_SESSION_ID] = testSessionId
       testSuiteTags[TEST_COMMAND] = this.command
     }

package/packages/datadog-plugin-mongodb-core/src/index.js

@@ -14,7 +14,7 @@ class MongodbCorePlugin extends Plugin {
 
     this.addSub(`apm:mongodb:query:start`, ({ ns, ops, options, name }) => {
       const query = getQuery(ops)
-      const resource = getResource(ns, query, name)
+      const resource = truncate(getResource(ns, query, name))
       const store = storage.getStore()
       const childOf = store ? store.span : store
       const span = this.tracer.startSpan('mongodb.query', {
@@ -55,8 +55,8 @@ class MongodbCorePlugin extends Plugin {
 
 function getQuery (cmd) {
   if (!cmd || typeof cmd !== 'object' || Array.isArray(cmd)) return
-  if (cmd.query) return JSON.stringify(sanitize(cmd.query))
-  if (cmd.filter) return JSON.stringify(sanitize(cmd.filter))
+  if (cmd.query) return JSON.stringify(limitDepth(cmd.query))
+  if (cmd.filter) return JSON.stringify(limitDepth(cmd.filter))
 }
 
 function getResource (ns, query, operationName) {
@@ -69,12 +69,25 @@ function getResource (ns, query, operationName) {
   return parts.join(' ')
 }
 
+function truncate (input) {
+  return input.slice(0, Math.min(input.length, 10000))
+}
+
+function simplify (input) {
+  return isBSON(input) ? input.toHexString() : input
+}
+
+function shouldSimplify (input) {
+  return !isObject(input) || isBSON(input)
+}
+
 function shouldHide (input) {
-  return !isObject(input) || Buffer.isBuffer(input) || isBSON(input)
+  return Buffer.isBuffer(input) || typeof input === 'function'
 }
 
-function sanitize (input) {
+function limitDepth (input) {
   if (shouldHide(input)) return '?'
+  if (shouldSimplify(input)) return simplify(input)
 
   const output = {}
   const queue = [{
@@ -92,8 +105,10 @@ function sanitize (input) {
       if (typeof input[key] === 'function') continue
 
       const child = input[key]
-      if (depth >= 20 || shouldHide(child)) {
+      if (depth >= 10 || shouldHide(child)) {
         output[key] = '?'
+      } else if (shouldSimplify(child)) {
+        output[key] = simplify(child)
       } else {
         queue.push({
           input: child,

package/packages/datadog-plugin-oracledb/src/index.js

@@ -3,6 +3,7 @@
 const Plugin = require('../../dd-trace/src/plugins/plugin')
 const { storage } = require('../../datadog-core')
 const analyticsSampler = require('../../dd-trace/src/analytics_sampler')
+const log = require('../../dd-trace/src/log')
 
 class OracledbPlugin extends Plugin {
   static get name () {
@@ -14,18 +15,25 @@ class OracledbPlugin extends Plugin {
 
     this.addSub('apm:oracledb:execute:start', ({ query, connAttrs }) => {
       const service = getServiceName(this.tracer, this.config, connAttrs)
-      const connectStringObj = new URL('http://' + connAttrs.connectString)
+      let connectStringObj
+      try {
+        connectStringObj = new URL(`http://${connAttrs.connectString}`)
+      } catch (e) {
+        log.error(e)
+      }
       const tags = {
         'span.kind': 'client',
         'span.type': 'sql',
         'sql.query': query,
-        'db.instance': connectStringObj.pathname.substring(1),
-        'db.hostname': connectStringObj.hostname,
         'db.user': this.config.user,
-        'db.port': connectStringObj.port,
         'resource.name': query,
         'service.name': service
       }
+      if (typeof connectStringObj !== 'undefined') {
+        tags['db.instance'] = connectStringObj.pathname.substring(1)
+        tags['db.hostname'] = connectStringObj.hostname
+        tags['db.port'] = connectStringObj.port
+      }
       const store = storage.getStore()
       const childOf = store ? store.span : store
       const span = this.tracer.startSpan('oracle.query', {

package/packages/datadog-plugin-restify/src/index.js

@@ -19,6 +19,13 @@ class RestifyPlugin extends RouterPlugin {
       web.setRoute(req, route)
     })
   }
+
+  configure (config) {
+    return super.configure({
+      ...config,
+      middleware: false // not supported
+    })
+  }
 }
 
 module.exports = RestifyPlugin

package/packages/dd-trace/index.js

@@ -1,7 +1,7 @@
 'use strict'
 
 if (!global._ddtrace) {
-  const TracerProxy = require('./src/proxy')
+  const TracerProxy = require('./src')
 
   Object.defineProperty(global, '_ddtrace', {
     value: new TracerProxy(),

package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js

@@ -0,0 +1,50 @@
+'use strict'
+const request = require('../../../exporters/common/request')
+const log = require('../../../log')
+
+const { CoverageCIVisibilityEncoder } = require('../../../encode/coverage-ci-visibility')
+const BaseWriter = require('../../../exporters/common/writer')
+
+function safeJSONStringify (value) {
+  return JSON.stringify(value, (key, value) =>
+    key !== 'dd-api-key' ? value : undefined
+  )
+}
+
+class Writer extends BaseWriter {
+  constructor ({ url }) {
+    super(...arguments)
+    this._url = url
+    this._encoder = new CoverageCIVisibilityEncoder()
+  }
+
+  _sendPayload (form, _, done) {
+    const options = {
+      path: '/api/v2/citestcov',
+      method: 'POST',
+      headers: {
+        'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY,
+        ...form.getHeaders()
+      },
+      timeout: 15000
+    }
+
+    options.protocol = this._url.protocol
+    options.hostname = this._url.hostname
+    options.port = this._url.port
+
+    log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
+
+    request(form, options, (err, res) => {
+      if (err) {
+        log.error(err)
+        done()
+        return
+      }
+      log.debug(`Response from the intake: ${res}`)
+      done()
+    })
+  }
+}
+
+module.exports = Writer

package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js

@@ -2,30 +2,75 @@
 
 const URL = require('url').URL
 const Writer = require('./writer')
-const Scheduler = require('../../../exporters/scheduler')
+const CoverageWriter = require('./coverage-writer')
+
+const log = require('../../../log')
 
 class AgentlessCiVisibilityExporter {
   constructor (config) {
-    const { flushInterval, tags, site, url } = config
+    this._config = config
+    const { tags, site, url, isIntelligentTestRunnerEnabled } = config
+    this._isIntelligentTestRunnerEnabled = isIntelligentTestRunnerEnabled
     this._url = url || new URL(`https://citestcycle-intake.${site}`)
     this._writer = new Writer({ url: this._url, tags })
+    this._timer = undefined
+    this._coverageTimer = undefined
+
+    this._coverageUrl = url || new URL(`https://event-platform-intake.${site}`)
+    this._coverageWriter = new CoverageWriter({ url: this._coverageUrl })
+
+    process.once('beforeExit', () => {
+      this._writer.flush()
+      this._coverageWriter.flush()
+    })
+  }
 
-    if (flushInterval > 0) {
-      this._scheduler = new Scheduler(() => this._writer.flush(), flushInterval)
+  exportCoverage ({ testSpan, coverageFiles }) {
+    const formattedCoverage = {
+      traceId: testSpan.context()._traceId,
+      spanId: testSpan.context()._spanId,
+      files: coverageFiles
+    }
+    this._coverageWriter.append(formattedCoverage)
+
+    const { flushInterval } = this._config
+
+    if (flushInterval === 0) {
+      this._coverageWriter.flush()
+    } else if (flushInterval > 0 && !this._coverageTimer) {
+      this._coverageTimer = setTimeout(() => {
+        this._coverageWriter.flush()
+        this._coverageTimer = clearTimeout(this._coverageTimer)
+      }, flushInterval).unref()
     }
-    this._scheduler && this._scheduler.start()
   }
 
   export (trace) {
     this._writer.append(trace)
 
-    if (!this._scheduler) {
+    const { flushInterval } = this._config
+
+    if (flushInterval === 0) {
       this._writer.flush()
+    } else if (flushInterval > 0 && !this._timer) {
+      this._timer = setTimeout(() => {
+        this._writer.flush()
+        this._timer = clearTimeout(this._timer)
+      }, flushInterval).unref()
     }
   }
 
-  flush () {
-    this._writer.flush()
+  setUrl (url, coverageUrl = url) {
+    try {
+      url = new URL(url)
+      coverageUrl = new URL(coverageUrl)
+      this._url = url
+      this._coverageUrl = coverageUrl
+      this._writer.setUrl(url)
+      this._coverageWriter.setUrl(coverageUrl)
+    } catch (e) {
+      log.error(e)
+    }
   }
 }
 

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -5,16 +5,37 @@ const log = require('../../../log')
 const { AgentlessCiVisibilityEncoder } = require('../../../encode/agentless-ci-visibility')
 const BaseWriter = require('../../../exporters/common/writer')
 
+function safeJSONStringify (value) {
+  return JSON.stringify(value, (key, value) =>
+    key !== 'dd-api-key' ? value : undefined
+  )
+}
+
 class Writer extends BaseWriter {
   constructor ({ url, tags }) {
     super(...arguments)
     const { 'runtime-id': runtimeId, env, service } = tags
     this._url = url
-    this._encoder = new AgentlessCiVisibilityEncoder({ runtimeId, env, service })
+    this._encoder = new AgentlessCiVisibilityEncoder(this, { runtimeId, env, service })
   }
 
   _sendPayload (data, _, done) {
-    makeRequest(data, this._url, (err, res) => {
+    const options = {
+      path: '/api/v2/citestcycle',
+      method: 'POST',
+      headers: {
+        'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY,
+        'Content-Type': 'application/msgpack'
+      },
+      timeout: 15000
+    }
+
+    options.protocol = this._url.protocol
+    options.hostname = this._url.hostname
+    options.port = this._url.port
+
+    log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
+    request(data, options, (err, res) => {
       if (err) {
         log.error(err)
         done()
@@ -26,26 +47,4 @@ class Writer extends BaseWriter {
   }
 }
 
-function makeRequest (data, url, cb) {
-  const options = {
-    path: '/api/v2/citestcycle',
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/msgpack',
-      'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY
-    },
-    timeout: 15000
-  }
-
-  options.protocol = url.protocol
-  options.hostname = url.hostname
-  options.port = url.port
-
-  log.debug(() => `Request to the intake: ${JSON.stringify(options)}`)
-
-  request(data, options, false, (err, res) => {
-    cb(err, res)
-  })
-}
-
 module.exports = Writer