dd-trace 3.0.0-pre.1 → 3.1.0

package/packages/datadog-instrumentations/src/jest.js

@@ -1,5 +1,5 @@
 'use strict'
-
+const istanbul = require('istanbul-lib-coverage')
 const { addHook, channel, AsyncResource } = require('./helpers/instrument')
 const shimmer = require('../../datadog-shimmer')
 
@@ -7,7 +7,8 @@ const testStartCh = channel('ci:jest:test:start')
 const testSkippedCh = channel('ci:jest:test:skip')
 const testRunFinishCh = channel('ci:jest:test:finish')
 const testErrCh = channel('ci:jest:test:err')
-const testSuiteFinish = channel('ci:jest:test-suite:finish')
+
+const testCodeCoverageCh = channel('ci:jest:test:code-coverage')
 
 const {
   getTestSuitePath,
@@ -16,6 +17,24 @@ const {
 
 const { getFormattedJestTestParameters, getJestTestName } = require('../../datadog-plugin-jest/src/util')
 
+// This function also resets the coverage counters
+function extractCoverageInformation (coverage, rootDir) {
+  const coverageMap = istanbul.createCoverageMap(coverage)
+
+  return coverageMap
+    .files()
+    .filter(filename => {
+      const fileCoverage = coverageMap.fileCoverageFor(filename)
+      const lineCoverage = fileCoverage.getLineCoverage()
+      const isAnyLineExecuted = Object.entries(lineCoverage).some(([, numExecutions]) => !!numExecutions)
+
+      fileCoverage.resetHits()
+
+      return isAnyLineExecuted
+    })
+    .map(filename => filename.replace(`${rootDir}/`, ''))
+}
+
 const specStatusToTestStatus = {
   'pending': 'skip',
   'disabled': 'skip',
@@ -49,15 +68,11 @@ function getWrappedEnvironment (BaseEnvironment) {
     constructor (config, context) {
       super(config, context)
       const rootDir = config.globalConfig ? config.globalConfig.rootDir : config.rootDir
+      this.rootDir = rootDir
       this.testSuite = getTestSuitePath(context.testPath, rootDir)
       this.nameToParams = {}
       this.global._ddtrace = global._ddtrace
     }
-    async teardown () {
-      super.teardown().finally(() => {
-        testSuiteFinish.publish()
-      })
-    }
 
     async handleTestEvent (event, state) {
       if (super.handleTestEvent) {
@@ -99,6 +114,10 @@ function getWrappedEnvironment (BaseEnvironment) {
       if (event.name === 'test_done') {
         const asyncResource = asyncResources.get(event.test)
         asyncResource.runInAsyncScope(() => {
+          if (this.global.__coverage__) {
+            const coverageFiles = extractCoverageInformation(this.global.__coverage__, this.rootDir)
+            testCodeCoverageCh.publish(coverageFiles)
+          }
           let status = 'pass'
           if (event.test.errors && event.test.errors.length) {
             status = 'fail'
@@ -111,10 +130,13 @@ function getWrappedEnvironment (BaseEnvironment) {
         })
       }
       if (event.name === 'test_skip' || event.name === 'test_todo') {
-        testSkippedCh.publish({
-          name: getJestTestName(event.test),
-          suite: this.testSuite,
-          runner: 'jest-circus'
+        const asyncResource = new AsyncResource('bound-anonymous-fn')
+        asyncResource.runInAsyncScope(() => {
+          testSkippedCh.publish({
+            name: getJestTestName(event.test),
+            suite: this.testSuite,
+            runner: 'jest-circus'
+          })
         })
       }
     }

package/packages/datadog-instrumentations/src/koa.js

@@ -145,7 +145,7 @@ function wrapNext (req, next) {
   return function () {
     nextChannel.publish({ req })
 
-    return next.apply(null, arguments)
+    return next.apply(this, arguments)
   }
 }
 

package/packages/datadog-instrumentations/src/mocha.js

@@ -153,7 +153,10 @@ function mochaHook (Runner) {
       if (isHook && testOrHook.ctx) {
         test = testOrHook.ctx.currentTest
       }
-      const asyncResource = getTestAsyncResource(test)
+      let asyncResource
+      if (test) {
+        asyncResource = getTestAsyncResource(test)
+      }
       if (asyncResource) {
         asyncResource.runInAsyncScope(() => {
           if (isHook) {

package/packages/datadog-instrumentations/src/pg.js

@@ -11,12 +11,12 @@ const startCh = channel('apm:pg:query:start')
 const finishCh = channel('apm:pg:query:finish')
 const errorCh = channel('apm:pg:query:error')
 
-addHook({ name: 'pg', versions: ['>=4.5.5'] }, pg => {
+addHook({ name: 'pg', versions: ['>=8.0.3'] }, pg => {
   shimmer.wrap(pg.Client.prototype, 'query', query => wrapQuery(query))
   return pg
 })
 
-addHook({ name: 'pg', file: 'lib/native/index.js', versions: ['>=4.5.5'] }, Client => {
+addHook({ name: 'pg', file: 'lib/native/index.js', versions: ['>=8.0.3'] }, Client => {
   shimmer.wrap(Client.prototype, 'query', query => wrapQuery(query))
   return Client
 })

package/packages/datadog-instrumentations/src/restify.js

@@ -6,7 +6,10 @@ const handlers = ['use', 'pre']
 const methods = ['del', 'get', 'head', 'opts', 'post', 'put', 'patch']
 
 const handleChannel = channel('apm:restify:request:handle')
-const routeChannel = channel('apm:restify:request:route')
+const errorChannel = channel('apm:restify:middleware:error')
+const enterChannel = channel('apm:restify:middleware:enter')
+const exitChannel = channel('apm:restify:middleware:exit')
+const nextChannel = channel('apm:restify:middleware:next')
 
 function wrapSetupRequest (setupRequest) {
   return function (req, res) {
@@ -37,18 +40,37 @@ function wrapFn (fn) {
   if (Array.isArray(fn)) return wrapMiddleware(fn)
 
   return function (req, res, next) {
-    if (req.route) {
-      routeChannel.publish({ req, route: req.route })
+    if (typeof next === 'function') {
+      arguments[2] = wrapNext(req, next)
     }
 
-    return fn.apply(this, arguments)
+    const route = req.route && req.route.path
+
+    enterChannel.publish({ req, route })
+
+    try {
+      return fn.apply(this, arguments)
+    } catch (error) {
+      errorChannel.publish({ req, error })
+      nextChannel.publish({ req })
+      exitChannel.publish({ req })
+    }
+  }
+}
+
+function wrapNext (req, next) {
+  return function () {
+    nextChannel.publish({ req })
+    exitChannel.publish({ req })
+
+    next.apply(this, arguments)
   }
 }
 
 addHook({ name: 'restify', versions: ['>=3'], file: 'lib/server.js' }, Server => {
   shimmer.wrap(Server.prototype, '_setupRequest', wrapSetupRequest)
   shimmer.massWrap(Server.prototype, handlers, wrapHandler)
-  shimmer.wrap(Server.prototype, methods, wrapMethod)
+  shimmer.massWrap(Server.prototype, methods, wrapMethod)
 
   return Server
 })

package/packages/datadog-instrumentations/src/router.js

@@ -97,7 +97,7 @@ function createWrapRouterMethod (name) {
       nextChannel.publish({ req })
       exitChannel.publish({ req })
 
-      next.apply(null, arguments)
+      next.apply(this, arguments)
     }
   }
 

package/packages/datadog-plugin-aws-sdk/src/base.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const Tags = require('opentracing').Tags
 const analyticsSampler = require('../../dd-trace/src/analytics_sampler')
 const Plugin = require('../../dd-trace/src/plugins/plugin')
 const { storage } = require('../../datadog-core')
@@ -33,7 +32,7 @@ class BaseAwsSdkPlugin extends Plugin {
       const serviceName = this.getServiceName(serviceIdentifier)
       const childOf = this.tracer.scope().active()
       const tags = {
-        [Tags.SPAN_KIND]: 'client',
+        'span.kind': 'client',
         'service.name': serviceName,
         'aws.operation': operation,
         'aws.region': awsRegion,

package/packages/datadog-plugin-aws-sdk/src/services/sqs.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const Tags = require('opentracing').Tags
 const log = require('../../../dd-trace/src/log')
 const BaseAwsSdkPlugin = require('../base')
 const { storage } = require('../../../datadog-core')
@@ -25,7 +24,7 @@ class Sqs extends BaseAwsSdkPlugin {
           tags: Object.assign(
             {},
             this.requestTags.get(request) || {},
-            { [Tags.SPAN_KIND]: 'server' }
+            { 'span.kind': 'server' }
           )
         }
         const span = plugin.tracer.startSpan('aws.response', options)

package/packages/datadog-plugin-cucumber/src/index.js

@@ -30,6 +30,10 @@ class CucumberPlugin extends Plugin {
     const sourceRoot = process.cwd()
     const codeOwnersEntries = getCodeOwnersFileEntries(sourceRoot)
 
+    this.addSub('ci:cucumber:session:finish', () => {
+      this.tracer._exporter._writer.flush()
+    })
+
     this.addSub('ci:cucumber:run:start', ({ pickleName, pickleUri }) => {
       const store = storage.getStore()
       const childOf = store ? store.span : store

package/packages/datadog-plugin-jest/src/index.js

@@ -15,6 +15,9 @@ const {
   TEST_CODE_OWNERS
 } = require('../../dd-trace/src/plugins/util/test')
 
+// https://github.com/facebook/jest/blob/d6ad15b0f88a05816c2fe034dd6900d28315d570/packages/jest-worker/src/types.ts#L38
+const CHILD_MESSAGE_END = 2
+
 function getTestSpanMetadata (tracer, test) {
   const childOf = getTestParentSpan(tracer)
 
@@ -38,9 +41,31 @@ class JestPlugin extends Plugin {
   constructor (...args) {
     super(...args)
 
+    // Used to handle the end of a jest worker to be able to flush
+    const handler = ([message]) => {
+      if (message === CHILD_MESSAGE_END) {
+        this.tracer._exporter._writer.flush(() => {
+          // eslint-disable-next-line
+          // https://github.com/facebook/jest/blob/24ed3b5ecb419c023ee6fdbc838f07cc028fc007/packages/jest-worker/src/workers/processChild.ts#L118-L133
+          // Only after the flush is done we clean up open handles
+          // so the worker process can hopefully exit gracefully
+          process.removeListener('message', handler)
+        })
+      }
+    }
+    process.on('message', handler)
+
     this.testEnvironmentMetadata = getTestEnvironmentMetadata('jest', this.config)
     this.codeOwnersEntries = getCodeOwnersFileEntries()
 
+    this.addSub('ci:jest:test:code-coverage', (coverageFiles) => {
+      if (!this.config.isAgentlessEnabled || !this.config.isIntelligentTestRunnerEnabled) {
+        return
+      }
+      const testSpan = storage.getStore().span
+      this.tracer._exporter.exportCoverage({ testSpan, coverageFiles })
+    })
+
     this.addSub('ci:jest:test:start', (test) => {
       const store = storage.getStore()
       const span = this.startTestSpan(test)
@@ -55,10 +80,6 @@ class JestPlugin extends Plugin {
       finishAllTraceSpans(span)
     })
 
-    this.addSub('ci:jest:test-suite:finish', () => {
-      this.tracer._exporter._writer.flush()
-    })
-
     this.addSub('ci:jest:test:err', (error) => {
       if (error) {
         const span = storage.getStore().span

package/packages/datadog-plugin-mocha/src/index.js

@@ -161,12 +161,12 @@ class MochaPlugin extends Plugin {
     const testSuiteSpan = this._testSuites.get(test.parent)
 
     if (testSuiteSpan) {
-      const testSuiteId = testSuiteSpan.context()._spanId.toString('hex')
+      const testSuiteId = testSuiteSpan.context()._spanId.toString(16)
       testSuiteTags[TEST_SUITE_ID] = testSuiteId
     }
 
     if (this.testSessionSpan) {
-      const testSessionId = this.testSessionSpan.context()._traceId.toString('hex')
+      const testSessionId = this.testSessionSpan.context()._traceId.toString(16)
       testSuiteTags[TEST_SESSION_ID] = testSessionId
       testSuiteTags[TEST_COMMAND] = this.command
     }

package/packages/datadog-plugin-mongodb-core/src/index.js

@@ -14,7 +14,7 @@ class MongodbCorePlugin extends Plugin {
 
     this.addSub(`apm:mongodb:query:start`, ({ ns, ops, options, name }) => {
       const query = getQuery(ops)
-      const resource = getResource(ns, query, name)
+      const resource = truncate(getResource(ns, query, name))
       const store = storage.getStore()
       const childOf = store ? store.span : store
       const span = this.tracer.startSpan('mongodb.query', {
@@ -55,8 +55,8 @@ class MongodbCorePlugin extends Plugin {
 
 function getQuery (cmd) {
   if (!cmd || typeof cmd !== 'object' || Array.isArray(cmd)) return
-  if (cmd.query) return JSON.stringify(sanitize(cmd.query))
-  if (cmd.filter) return JSON.stringify(sanitize(cmd.filter))
+  if (cmd.query) return JSON.stringify(limitDepth(cmd.query))
+  if (cmd.filter) return JSON.stringify(limitDepth(cmd.filter))
 }
 
 function getResource (ns, query, operationName) {
@@ -69,12 +69,25 @@ function getResource (ns, query, operationName) {
   return parts.join(' ')
 }
 
+function truncate (input) {
+  return input.slice(0, Math.min(input.length, 10000))
+}
+
+function simplify (input) {
+  return isBSON(input) ? input.toHexString() : input
+}
+
+function shouldSimplify (input) {
+  return !isObject(input) || isBSON(input)
+}
+
 function shouldHide (input) {
-  return !isObject(input) || Buffer.isBuffer(input) || isBSON(input)
+  return Buffer.isBuffer(input) || typeof input === 'function'
 }
 
-function sanitize (input) {
+function limitDepth (input) {
   if (shouldHide(input)) return '?'
+  if (shouldSimplify(input)) return simplify(input)
 
   const output = {}
   const queue = [{
@@ -92,8 +105,10 @@ function sanitize (input) {
       if (typeof input[key] === 'function') continue
 
       const child = input[key]
-      if (depth >= 20 || shouldHide(child)) {
+      if (depth >= 10 || shouldHide(child)) {
         output[key] = '?'
+      } else if (shouldSimplify(child)) {
+        output[key] = simplify(child)
       } else {
         queue.push({
           input: child,

package/packages/datadog-plugin-oracledb/src/index.js

@@ -3,6 +3,7 @@
 const Plugin = require('../../dd-trace/src/plugins/plugin')
 const { storage } = require('../../datadog-core')
 const analyticsSampler = require('../../dd-trace/src/analytics_sampler')
+const log = require('../../dd-trace/src/log')
 
 class OracledbPlugin extends Plugin {
   static get name () {
@@ -14,18 +15,25 @@ class OracledbPlugin extends Plugin {
 
     this.addSub('apm:oracledb:execute:start', ({ query, connAttrs }) => {
       const service = getServiceName(this.tracer, this.config, connAttrs)
-      const connectStringObj = new URL('http://' + connAttrs.connectString)
+      let connectStringObj
+      try {
+        connectStringObj = new URL(`http://${connAttrs.connectString}`)
+      } catch (e) {
+        log.error(e)
+      }
       const tags = {
         'span.kind': 'client',
         'span.type': 'sql',
         'sql.query': query,
-        'db.instance': connectStringObj.pathname.substring(1),
-        'db.hostname': connectStringObj.hostname,
         'db.user': this.config.user,
-        'db.port': connectStringObj.port,
         'resource.name': query,
         'service.name': service
       }
+      if (typeof connectStringObj !== 'undefined') {
+        tags['db.instance'] = connectStringObj.pathname.substring(1)
+        tags['db.hostname'] = connectStringObj.hostname
+        tags['db.port'] = connectStringObj.port
+      }
       const store = storage.getStore()
       const childOf = store ? store.span : store
       const span = this.tracer.startSpan('oracle.query', {

package/packages/datadog-plugin-restify/src/index.js

@@ -19,6 +19,13 @@ class RestifyPlugin extends RouterPlugin {
       web.setRoute(req, route)
     })
   }
+
+  configure (config) {
+    return super.configure({
+      ...config,
+      middleware: false // not supported
+    })
+  }
 }
 
 module.exports = RestifyPlugin

package/packages/dd-trace/index.js

@@ -1,7 +1,7 @@
 'use strict'
 
 if (!global._ddtrace) {
-  const TracerProxy = require('./src/proxy')
+  const TracerProxy = require('./src')
 
   Object.defineProperty(global, '_ddtrace', {
     value: new TracerProxy(),

package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js

@@ -0,0 +1,50 @@
+'use strict'
+const request = require('../../../exporters/common/request')
+const log = require('../../../log')
+
+const { CoverageCIVisibilityEncoder } = require('../../../encode/coverage-ci-visibility')
+const BaseWriter = require('../../../exporters/common/writer')
+
+function safeJSONStringify (value) {
+  return JSON.stringify(value, (key, value) =>
+    key !== 'dd-api-key' ? value : undefined
+  )
+}
+
+class Writer extends BaseWriter {
+  constructor ({ url }) {
+    super(...arguments)
+    this._url = url
+    this._encoder = new CoverageCIVisibilityEncoder()
+  }
+
+  _sendPayload (form, _, done) {
+    const options = {
+      path: '/api/v2/citestcov',
+      method: 'POST',
+      headers: {
+        'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY,
+        ...form.getHeaders()
+      },
+      timeout: 15000
+    }
+
+    options.protocol = this._url.protocol
+    options.hostname = this._url.hostname
+    options.port = this._url.port
+
+    log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
+
+    request(form, options, (err, res) => {
+      if (err) {
+        log.error(err)
+        done()
+        return
+      }
+      log.debug(`Response from the intake: ${res}`)
+      done()
+    })
+  }
+}
+
+module.exports = Writer

package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js

@@ -2,30 +2,75 @@
 
 const URL = require('url').URL
 const Writer = require('./writer')
-const Scheduler = require('../../../exporters/scheduler')
+const CoverageWriter = require('./coverage-writer')
+
+const log = require('../../../log')
 
 class AgentlessCiVisibilityExporter {
   constructor (config) {
-    const { flushInterval, tags, site, url } = config
+    this._config = config
+    const { tags, site, url, isIntelligentTestRunnerEnabled } = config
+    this._isIntelligentTestRunnerEnabled = isIntelligentTestRunnerEnabled
     this._url = url || new URL(`https://citestcycle-intake.${site}`)
     this._writer = new Writer({ url: this._url, tags })
+    this._timer = undefined
+    this._coverageTimer = undefined
+
+    this._coverageUrl = url || new URL(`https://event-platform-intake.${site}`)
+    this._coverageWriter = new CoverageWriter({ url: this._coverageUrl })
+
+    process.once('beforeExit', () => {
+      this._writer.flush()
+      this._coverageWriter.flush()
+    })
+  }
 
-    if (flushInterval > 0) {
-      this._scheduler = new Scheduler(() => this._writer.flush(), flushInterval)
+  exportCoverage ({ testSpan, coverageFiles }) {
+    const formattedCoverage = {
+      traceId: testSpan.context()._traceId,
+      spanId: testSpan.context()._spanId,
+      files: coverageFiles
+    }
+    this._coverageWriter.append(formattedCoverage)
+
+    const { flushInterval } = this._config
+
+    if (flushInterval === 0) {
+      this._coverageWriter.flush()
+    } else if (flushInterval > 0 && !this._coverageTimer) {
+      this._coverageTimer = setTimeout(() => {
+        this._coverageWriter.flush()
+        this._coverageTimer = clearTimeout(this._coverageTimer)
+      }, flushInterval).unref()
     }
-    this._scheduler && this._scheduler.start()
   }
 
   export (trace) {
     this._writer.append(trace)
 
-    if (!this._scheduler) {
+    const { flushInterval } = this._config
+
+    if (flushInterval === 0) {
       this._writer.flush()
+    } else if (flushInterval > 0 && !this._timer) {
+      this._timer = setTimeout(() => {
+        this._writer.flush()
+        this._timer = clearTimeout(this._timer)
+      }, flushInterval).unref()
     }
   }
 
-  flush () {
-    this._writer.flush()
+  setUrl (url, coverageUrl = url) {
+    try {
+      url = new URL(url)
+      coverageUrl = new URL(coverageUrl)
+      this._url = url
+      this._coverageUrl = coverageUrl
+      this._writer.setUrl(url)
+      this._coverageWriter.setUrl(coverageUrl)
+    } catch (e) {
+      log.error(e)
+    }
   }
 }
 

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -5,16 +5,37 @@ const log = require('../../../log')
 const { AgentlessCiVisibilityEncoder } = require('../../../encode/agentless-ci-visibility')
 const BaseWriter = require('../../../exporters/common/writer')
 
+function safeJSONStringify (value) {
+  return JSON.stringify(value, (key, value) =>
+    key !== 'dd-api-key' ? value : undefined
+  )
+}
+
 class Writer extends BaseWriter {
   constructor ({ url, tags }) {
     super(...arguments)
     const { 'runtime-id': runtimeId, env, service } = tags
     this._url = url
-    this._encoder = new AgentlessCiVisibilityEncoder({ runtimeId, env, service })
+    this._encoder = new AgentlessCiVisibilityEncoder(this, { runtimeId, env, service })
   }
 
   _sendPayload (data, _, done) {
-    makeRequest(data, this._url, (err, res) => {
+    const options = {
+      path: '/api/v2/citestcycle',
+      method: 'POST',
+      headers: {
+        'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY,
+        'Content-Type': 'application/msgpack'
+      },
+      timeout: 15000
+    }
+
+    options.protocol = this._url.protocol
+    options.hostname = this._url.hostname
+    options.port = this._url.port
+
+    log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
+    request(data, options, (err, res) => {
       if (err) {
         log.error(err)
         done()
@@ -26,26 +47,4 @@ class Writer extends BaseWriter {
   }
 }
 
-function makeRequest (data, url, cb) {
-  const options = {
-    path: '/api/v2/citestcycle',
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/msgpack',
-      'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY
-    },
-    timeout: 15000
-  }
-
-  options.protocol = url.protocol
-  options.hostname = url.hostname
-  options.port = url.port
-
-  log.debug(() => `Request to the intake: ${JSON.stringify(options)}`)
-
-  request(data, options, false, (err, res) => {
-    cb(err, res)
-  })
-}
-
 module.exports = Writer