dd-trace 2.21.0 → 2.22.1

package/packages/datadog-instrumentations/src/jest.js

@@ -1,8 +1,8 @@
 'use strict'
-const istanbul = require('istanbul-lib-coverage')
 const { addHook, channel, AsyncResource } = require('./helpers/instrument')
 const shimmer = require('../../datadog-shimmer')
 const log = require('../../dd-trace/src/log')
+const { getCoveredFilenamesFromCoverage } = require('../../dd-trace/src/plugins/util/test')
 
 const testSessionStartCh = channel('ci:jest:session:start')
 const testSessionFinishCh = channel('ci:jest:session:finish')
@@ -23,6 +23,7 @@ const jestConfigurationCh = channel('ci:jest:configuration')
 
 let skippableSuites = []
 let isCodeCoverageEnabled = false
+let isSuitesSkippingEnabled = false
 
 const {
   getTestSuitePath,
@@ -33,21 +34,6 @@ const { getFormattedJestTestParameters, getJestTestName } = require('../../datad
 
 const sessionAsyncResource = new AsyncResource('bound-anonymous-fn')
 
-function extractCoverageInformation (coverage, rootDir) {
-  const coverageMap = istanbul.createCoverageMap(coverage)
-
-  return coverageMap
-    .files()
-    .filter(filename => {
-      const fileCoverage = coverageMap.fileCoverageFor(filename)
-      const lineCoverage = fileCoverage.getLineCoverage()
-      const isAnyLineExecuted = Object.entries(lineCoverage).some(([, numExecutions]) => !!numExecutions)
-
-      return isAnyLineExecuted
-    })
-    .map(filename => filename.replace(`${rootDir}/`, ''))
-}
-
 const specStatusToTestStatus = {
   'pending': 'skip',
   'disabled': 'skip',
@@ -186,38 +172,42 @@ addHook({
 
 function cliWrapper (cli) {
   const wrapped = shimmer.wrap(cli, 'runCLI', runCLI => async function () {
-    let onResponse, onError
-    const configurationPromise = new Promise((resolve, reject) => {
-      onResponse = resolve
-      onError = reject
+    let onDone
+    const configurationPromise = new Promise((resolve) => {
+      onDone = resolve
     })
 
     sessionAsyncResource.runInAsyncScope(() => {
-      jestConfigurationCh.publish({ onResponse, onError })
+      jestConfigurationCh.publish({ onDone })
     })
 
-    let isSuitesSkippingEnabled = false
-
     try {
-      const config = await configurationPromise
+      const { err, config } = await configurationPromise
+      if (err) {
+        log.error(err)
+      }
       isCodeCoverageEnabled = config.isCodeCoverageEnabled
       isSuitesSkippingEnabled = config.isSuitesSkippingEnabled
     } catch (e) {
-      // ignore error
+      log.error(e)
     }
 
     if (isSuitesSkippingEnabled) {
-      const skippableSuitesPromise = new Promise((resolve, reject) => {
-        onResponse = resolve
-        onError = reject
+      const skippableSuitesPromise = new Promise((resolve) => {
+        onDone = resolve
       })
 
       sessionAsyncResource.runInAsyncScope(() => {
-        skippableSuitesCh.publish({ onResponse, onError })
+        skippableSuitesCh.publish({ onDone })
       })
 
       try {
-        skippableSuites = await skippableSuitesPromise
+        const { err, skippableSuites: receivedSkippableSuites } = await skippableSuitesPromise
+        if (err) {
+          log.error(err)
+        } else {
+          skippableSuites = receivedSkippableSuites
+        }
       } catch (e) {
         log.error(e)
       }
@@ -253,6 +243,35 @@ function cliWrapper (cli) {
   return cli
 }
 
+function coverageReporterWrapper (coverageReporter) {
+  const CoverageReporter = coverageReporter.default ? coverageReporter.default : coverageReporter
+
+  /**
+   * If ITR is active, we're running fewer tests, so of course the total code coverage is reduced.
+   * This calculation adds no value, so we'll skip it.
+   */
+  shimmer.wrap(CoverageReporter.prototype, '_addUntestedFiles', addUntestedFiles => async function () {
+    if (isSuitesSkippingEnabled) {
+      return Promise.resolve()
+    }
+    return addUntestedFiles.apply(this, arguments)
+  })
+
+  return coverageReporter
+}
+
+addHook({
+  name: '@jest/reporters',
+  file: 'build/coverage_reporter.js',
+  versions: ['>=24.8.0 <26.6.2']
+}, coverageReporterWrapper)
+
+addHook({
+  name: '@jest/reporters',
+  file: 'build/CoverageReporter.js',
+  versions: ['>=26.6.2']
+}, coverageReporterWrapper)
+
 addHook({
   name: '@jest/core',
   file: 'build/cli/index.js',
@@ -281,13 +300,16 @@ function jestAdapterWrapper (jestAdapter) {
           status = 'fail'
         }
         testSuiteFinishCh.publish({ status, errorMessage })
-        if (environment.global.__coverage__) {
-          const coverageFiles = extractCoverageInformation(environment.global.__coverage__, environment.rootDir)
-          if (coverageFiles.length &&
-            environment.testEnvironmentOptions &&
-            environment.testEnvironmentOptions._ddTestCodeCoverageEnabled) {
+
+        const coverageFiles = getCoveredFilenamesFromCoverage(environment.global.__coverage__)
+          .map(filename => getTestSuitePath(filename, environment.rootDir))
+
+        if (coverageFiles &&
+          environment.testEnvironmentOptions &&
+          environment.testEnvironmentOptions._ddTestCodeCoverageEnabled) {
+          asyncResource.runInAsyncScope(() => {
             testSuiteCodeCoverageCh.publish([...coverageFiles, environment.testSuite])
-          }
+          })
         }
         return suiteResults
       })
@@ -310,12 +332,6 @@ addHook({
 
 function configureTestEnvironment (readConfigsResult) {
   const { configs } = readConfigsResult
-  configs.forEach(config => {
-    skippableSuites.forEach((suite) => {
-      config.testMatch.push(`!**/${suite}`)
-    })
-    skippableSuites = []
-  })
   sessionAsyncResource.runInAsyncScope(() => {
     testSessionConfigurationCh.publish(configs.map(config => config.testEnvironmentOptions))
   })
@@ -324,6 +340,7 @@ function configureTestEnvironment (readConfigsResult) {
   configs.forEach(config => {
     config.testEnvironmentOptions._ddTestCodeCoverageEnabled = isCodeCoverageEnabled
   })
+
   if (isCodeCoverageEnabled) {
     const globalConfig = {
       ...readConfigsResult.globalConfig,
@@ -331,6 +348,16 @@ function configureTestEnvironment (readConfigsResult) {
     }
     readConfigsResult.globalConfig = globalConfig
   }
+  if (isSuitesSkippingEnabled) {
+    // If suite skipping is enabled, the code coverage results are not going to be relevant,
+    // so we do not show them.
+    const globalConfig = {
+      ...readConfigsResult.globalConfig,
+      coverageReporters: ['none']
+    }
+    readConfigsResult.globalConfig = globalConfig
+  }
+
   return readConfigsResult
 }
 
@@ -352,6 +379,39 @@ function jestConfigSyncWrapper (jestConfig) {
   return jestConfig
 }
 
+/**
+ * Hook to remove the test paths (test suite) that are part of `skippableSuites`
+ */
+addHook({
+  name: '@jest/core',
+  versions: ['>=24.8.0'],
+  file: 'build/SearchSource.js'
+}, searchSourcePackage => {
+  const SearchSource = searchSourcePackage.default ? searchSourcePackage.default : searchSourcePackage
+
+  shimmer.wrap(SearchSource.prototype, 'getTestPaths', getTestPaths => async function () {
+    if (!skippableSuites.length) {
+      return getTestPaths.apply(this, arguments)
+    }
+
+    const [{ rootDir }] = arguments
+
+    const testPaths = await getTestPaths.apply(this, arguments)
+    const { tests } = testPaths
+
+    const filteredTests = tests.filter(({ path: testPath }) => {
+      const relativePath = testPath.replace(`${rootDir}/`, '')
+      return !skippableSuites.includes(relativePath)
+    })
+
+    skippableSuites = []
+
+    return { ...testPaths, tests: filteredTests }
+  })
+
+  return searchSourcePackage
+})
+
 // from 25.1.0 on, readConfigs becomes async
 addHook({
   name: 'jest-config',

package/packages/datadog-instrumentations/src/koa.js

@@ -1,12 +1,13 @@
 'use strict'
 
 const shimmer = require('../../datadog-shimmer')
-const { addHook, channel, AsyncResource } = require('./helpers/instrument')
+const { addHook, channel } = require('./helpers/instrument')
 
 const enterChannel = channel('apm:koa:middleware:enter')
 const exitChannel = channel('apm:koa:middleware:exit')
 const errorChannel = channel('apm:koa:middleware:error')
 const nextChannel = channel('apm:koa:middleware:next')
+const finishChannel = channel('apm:koa:middleware:finish')
 const handleChannel = channel('apm:koa:request:handle')
 const routeChannel = channel('apm:koa:request:route')
 
@@ -86,42 +87,41 @@ function wrapMiddleware (fn, layer) {
   return function (ctx, next) {
     if (!ctx || !enterChannel.hasSubscribers) return fn.apply(this, arguments)
 
-    const middlewareResource = new AsyncResource('bound-anonymous-fn')
     const req = ctx.req
 
-    return middlewareResource.runInAsyncScope(() => {
-      const path = layer && layer.path
-      const route = typeof path === 'string' && !path.endsWith('(.*)') && !path.endsWith('([^/]*)') && path
+    const path = layer && layer.path
+    const route = typeof path === 'string' && !path.endsWith('(.*)') && !path.endsWith('([^/]*)') && path
 
-      enterChannel.publish({ req, name, route })
+    enterChannel.publish({ req, name, route })
 
-      if (typeof next === 'function') {
-        arguments[1] = wrapNext(req, next)
-      }
+    if (typeof next === 'function') {
+      arguments[1] = wrapNext(req, next)
+    }
 
-      try {
-        const result = fn.apply(this, arguments)
-
-        if (result && typeof result.then === 'function') {
-          return result.then(
-            result => {
-              fulfill(ctx)
-              return result
-            },
-            err => {
-              fulfill(ctx, err)
-              throw err
-            }
-          )
-        } else {
-          fulfill(ctx)
-          return result
-        }
-      } catch (e) {
-        fulfill(ctx, e)
-        throw e
+    try {
+      const result = fn.apply(this, arguments)
+
+      if (result && typeof result.then === 'function') {
+        return result.then(
+          result => {
+            fulfill(ctx)
+            return result
+          },
+          err => {
+            fulfill(ctx, err)
+            throw err
+          }
+        )
+      } else {
+        fulfill(ctx)
+        return result
       }
-    })
+    } catch (e) {
+      fulfill(ctx, e)
+      throw e
+    } finally {
+      exitChannel.publish({ req })
+    }
   }
 }
 
@@ -138,7 +138,7 @@ function fulfill (ctx, error) {
     routeChannel.publish({ req, route })
   }
 
-  exitChannel.publish({ req })
+  finishChannel.publish({ req })
 }
 
 function wrapNext (req, next) {

package/packages/datadog-instrumentations/src/mocha.js

@@ -1,5 +1,15 @@
+const { createCoverageMap } = require('istanbul-lib-coverage')
+
 const { addHook, channel, AsyncResource } = require('./helpers/instrument')
 const shimmer = require('../../datadog-shimmer')
+const log = require('../../dd-trace/src/log')
+const {
+  getCoveredFilenamesFromCoverage,
+  resetCoverage,
+  mergeCoverage,
+  getTestSuitePath,
+  fromCoverageMapToCoverage
+} = require('../../dd-trace/src/plugins/util/test')
 
 const testStartCh = channel('ci:mocha:test:start')
 const errorCh = channel('ci:mocha:test:error')
@@ -7,12 +17,16 @@ const skipCh = channel('ci:mocha:test:skip')
 const testFinishCh = channel('ci:mocha:test:finish')
 const parameterizedTestCh = channel('ci:mocha:test:parameterize')
 
+const configurationCh = channel('ci:mocha:configuration')
+const skippableSuitesCh = channel('ci:mocha:test-suite:skippable')
+
 const testSessionStartCh = channel('ci:mocha:session:start')
 const testSessionFinishCh = channel('ci:mocha:session:finish')
 
 const testSuiteStartCh = channel('ci:mocha:test-suite:start')
 const testSuiteFinishCh = channel('ci:mocha:test-suite:finish')
 const testSuiteErrorCh = channel('ci:mocha:test-suite:error')
+const testSuiteCodeCoverageCh = channel('ci:mocha:test-suite:code-coverage')
 
 // TODO: remove when root hooks and fixtures are implemented
 const patched = new WeakSet()
@@ -21,6 +35,11 @@ const testToAr = new WeakMap()
 const originalFns = new WeakMap()
 const testFileToSuiteAr = new Map()
 
+// We'll preserve the original coverage here
+const originalCoverageMap = createCoverageMap()
+
+let suitesToSkip = []
+
 function getSuitesByTestFile (root) {
   const suitesByTestFile = {}
   function getSuites (suite) {
@@ -46,13 +65,13 @@ function getSuitesByTestFile (root) {
 }
 
 function getTestStatus (test) {
-  if (test.pending) {
+  if (test.isPending()) {
     return 'skip'
   }
-  if (test.state !== 'failed' && !test.timedOut) {
-    return 'pass'
+  if (test.isFailed() || test.timedOut) {
+    return 'fail'
   }
-  return 'fail'
+  return 'pass'
 }
 
 function isRetry (test) {
@@ -93,6 +112,8 @@ function mochaHook (Runner) {
       }
       testFileToSuiteAr.clear()
       testSessionFinishCh.publish(status)
+      // restore the original coverage
+      global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap)
     }))
 
     this.once('start', testRunAsyncResource.bind(function () {
@@ -140,9 +161,21 @@ function mochaHook (Runner) {
         })
       }
 
+      if (global.__coverage__) {
+        const coverageFiles = getCoveredFilenamesFromCoverage(global.__coverage__)
+
+        testSuiteCodeCoverageCh.publish({
+          coverageFiles,
+          suiteFile: suite.file
+        })
+        // We need to reset coverage to get a code coverage per suite
+        // Before that, we preserve the original coverage
+        mergeCoverage(global.__coverage__, originalCoverageMap)
+        resetCoverage(global.__coverage__)
+      }
+
       const asyncResource = testFileToSuiteAr.get(suite.file)
       asyncResource.runInAsyncScope(() => {
-        // get suite status
         testSuiteFinishCh.publish(status)
       })
     })
@@ -229,7 +262,8 @@ function mochaHook (Runner) {
           skipCh.publish(test)
         })
       } else {
-        // if there is no async resource, the test has been skipped through `test.skip``
+        // if there is no async resource, the test has been skipped through `test.skip`
+        // or the parent suite is skipped
         const skippedTestAsyncResource = new AsyncResource('bound-anonymous-fn')
         if (test.fn) {
           testToAr.set(test.fn, skippedTestAsyncResource)
@@ -242,6 +276,11 @@ function mochaHook (Runner) {
       }
     })
 
+    // We remove the suites that we skip through ITR
+    this.suite.suites = this.suite.suites.filter(suite =>
+      !suitesToSkip.includes(getTestSuitePath(suite.file, process.cwd()))
+    )
+
     return run.apply(this, arguments)
   })
 
@@ -266,6 +305,48 @@ function mochaEachHook (mochaEach) {
   })
 }
 
+addHook({
+  name: 'mocha',
+  versions: ['>=5.2.0'],
+  file: 'lib/mocha.js'
+}, (Mocha) => {
+  const mochaRunAsyncResource = new AsyncResource('bound-anonymous-fn')
+
+  /**
+   * Get ITR configuration and skippable suites
+   * If ITR is disabled, `onDone` is called immediately on the subscriber
+   */
+  shimmer.wrap(Mocha.prototype, 'run', run => function () {
+    const onReceivedSkippableSuites = ({ err, skippableSuites }) => {
+      if (err) {
+        log.error(err)
+        suitesToSkip = []
+      } else {
+        suitesToSkip = skippableSuites
+      }
+      run.apply(this, arguments)
+    }
+
+    const onReceivedConfiguration = ({ err }) => {
+      if (err) {
+        log.error(err)
+        return run.apply(this, arguments)
+      }
+
+      skippableSuitesCh.publish({
+        onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites)
+      })
+    }
+
+    mochaRunAsyncResource.runInAsyncScope(() => {
+      configurationCh.publish({
+        onDone: mochaRunAsyncResource.bind(onReceivedConfiguration)
+      })
+    })
+  })
+  return Mocha
+})
+
 addHook({
   name: 'mocha',
   versions: ['>=5.2.0'],

package/packages/datadog-instrumentations/src/pg.js

@@ -37,13 +37,12 @@ function wrapQuery (query) {
       return retval
     }
 
-    const statement = pgQuery.text
     const callbackResource = new AsyncResource('bound-anonymous-fn')
     const asyncResource = new AsyncResource('bound-anonymous-fn')
     const processId = this.processID
 
     return asyncResource.runInAsyncScope(() => {
-      startCh.publish({ params: this.connectionParameters, statement, processId })
+      startCh.publish({ params: this.connectionParameters, query: pgQuery, processId })
 
       const finish = asyncResource.bind(function (error) {
         if (error) {

package/packages/datadog-instrumentations/src/qs.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const { addHook, channel } = require('./helpers/instrument')
+const shimmer = require('../../datadog-shimmer')
+
+const qsParseCh = channel('datadog:qs:parse:finish')
+
+function wrapParse (originalParse) {
+  return function () {
+    const qsParsedObj = originalParse.apply(this, arguments)
+    if (qsParseCh.hasSubscribers && qsParsedObj) {
+      qsParseCh.publish({ qs: qsParsedObj })
+    }
+    return qsParsedObj
+  }
+}
+
+addHook({
+  name: 'qs',
+  versions: ['>=1']
+}, qs => {
+  shimmer.wrap(qs, 'parse', wrapParse)
+  return qs
+})

package/packages/datadog-instrumentations/src/restify.js

@@ -9,6 +9,7 @@ const handleChannel = channel('apm:restify:request:handle')
 const errorChannel = channel('apm:restify:middleware:error')
 const enterChannel = channel('apm:restify:middleware:enter')
 const exitChannel = channel('apm:restify:middleware:exit')
+const finishChannel = channel('apm:restify:middleware:finish')
 const nextChannel = channel('apm:restify:middleware:next')
 
 function wrapSetupRequest (setupRequest) {
@@ -53,8 +54,10 @@ function wrapFn (fn) {
     } catch (error) {
       errorChannel.publish({ req, error })
       nextChannel.publish({ req })
-      exitChannel.publish({ req })
+      finishChannel.publish({ req })
       throw error
+    } finally {
+      exitChannel.publish({ req })
     }
   }
 }
@@ -62,13 +65,13 @@ function wrapFn (fn) {
 function wrapNext (req, next) {
   return function () {
     nextChannel.publish({ req })
-    exitChannel.publish({ req })
+    finishChannel.publish({ req })
 
     next.apply(this, arguments)
   }
 }
 
-addHook({ name: 'restify', versions: ['>=3'], file: 'lib/server.js' }, Server => {
+addHook({ name: 'restify', versions: ['>=3 <=8.6.1'], file: 'lib/server.js' }, Server => {
   shimmer.wrap(Server.prototype, '_setupRequest', wrapSetupRequest)
   shimmer.massWrap(Server.prototype, handlers, wrapHandler)
   shimmer.massWrap(Server.prototype, methods, wrapMethod)

package/packages/datadog-instrumentations/src/router.js

@@ -3,11 +3,12 @@
 const METHODS = require('methods').concat('all')
 const pathToRegExp = require('path-to-regexp')
 const shimmer = require('../../datadog-shimmer')
-const { addHook, channel, AsyncResource } = require('./helpers/instrument')
+const { addHook, channel } = require('./helpers/instrument')
 
 function createWrapRouterMethod (name) {
   const enterChannel = channel(`apm:${name}:middleware:enter`)
   const exitChannel = channel(`apm:${name}:middleware:exit`)
+  const finishChannel = channel(`apm:${name}:middleware:finish`)
   const errorChannel = channel(`apm:${name}:middleware:error`)
   const nextChannel = channel(`apm:${name}:middleware:next`)
 
@@ -21,7 +22,6 @@ function createWrapRouterMethod (name) {
       if (!enterChannel.hasSubscribers) return original.apply(this, arguments)
 
       const matchers = layerMatchers.get(layer)
-      const middlewareResource = new AsyncResource('bound-anonymous-fn')
       const lastIndex = arguments.length - 1
       const name = original._name || original.name
       const req = arguments[arguments.length > 3 ? 1 : 0]
@@ -31,32 +31,32 @@ function createWrapRouterMethod (name) {
         arguments[lastIndex] = wrapNext(req, next)
       }
 
-      return middlewareResource.runInAsyncScope(() => {
-        let route
+      let route
 
-        if (matchers) {
-          // Try to guess which path actually matched
-          for (let i = 0; i < matchers.length; i++) {
-            if (matchers[i].test(layer)) {
-              route = matchers[i].path
+      if (matchers) {
+        // Try to guess which path actually matched
+        for (let i = 0; i < matchers.length; i++) {
+          if (matchers[i].test(layer)) {
+            route = matchers[i].path
 
-              break
-            }
+            break
           }
         }
+      }
 
-        enterChannel.publish({ name, req, route })
+      enterChannel.publish({ name, req, route })
 
-        try {
-          return original.apply(this, arguments)
-        } catch (error) {
-          errorChannel.publish({ req, error })
-          nextChannel.publish({ req })
-          exitChannel.publish({ req })
+      try {
+        return original.apply(this, arguments)
+      } catch (error) {
+        errorChannel.publish({ req, error })
+        nextChannel.publish({ req })
+        finishChannel.publish({ req })
 
-          throw error
-        }
-      })
+        throw error
+      } finally {
+        exitChannel.publish({ req })
+      }
     })
 
     // This is a workaround for the `loopback` library so that it can find the correct express layer
@@ -95,7 +95,7 @@ function createWrapRouterMethod (name) {
       }
 
       nextChannel.publish({ req })
-      exitChannel.publish({ req })
+      finishChannel.publish({ req })
 
       next.apply(this, arguments)
     }