dd-trace 5.11.0 → 5.13.0

package/ci/init.js

@@ -3,6 +3,7 @@ const tracer = require('../packages/dd-trace')
 const { isTrue } = require('../packages/dd-trace/src/util')
 
 const isJestWorker = !!process.env.JEST_WORKER_ID
+const isCucumberWorker = !!process.env.CUCUMBER_WORKER_ID
 
 const options = {
   startupLogs: false,
@@ -37,6 +38,12 @@ if (isJestWorker) {
   }
 }
 
+if (isCucumberWorker) {
+  options.experimental = {
+    exporter: 'cucumber_worker'
+  }
+}
+
 if (shouldInit) {
   tracer.init(options)
   tracer.use('fs', false)

package/ext/exporters.d.ts

@@ -3,7 +3,8 @@ declare const exporters: {
   AGENT: 'agent',
   DATADOG: 'datadog',
   AGENT_PROXY: 'agent_proxy',
-  JEST_WORKER: 'jest_worker'
+  JEST_WORKER: 'jest_worker',
+  CUCUMBER_WORKER: 'cucumber_worker'
 }
 
 export = exporters

package/ext/exporters.js

@@ -4,5 +4,6 @@ module.exports = {
   AGENT: 'agent',
   DATADOG: 'datadog',
   AGENT_PROXY: 'agent_proxy',
-  JEST_WORKER: 'jest_worker'
+  JEST_WORKER: 'jest_worker',
+  CUCUMBER_WORKER: 'cucumber_worker'
 }

package/index.d.ts

@@ -940,11 +940,15 @@ declare namespace tracer {
     /** @hidden */
     interface Http extends Instrumentation {
       /**
-       * List of URLs that should be instrumented.
+       * List of URLs/paths that should be instrumented.
+       *
+       * Note that when used for an http client the entry represents a full
+       * outbound URL (`https://example.org/api/foo`) but when used as a
+       * server the entry represents an inbound path (`/api/foo`).
        *
        * @default /^.*$/
        */
-      allowlist?: string | RegExp | ((url: string) => boolean) | (string | RegExp | ((url: string) => boolean))[];
+      allowlist?: string | RegExp | ((urlOrPath: string) => boolean) | (string | RegExp | ((urlOrPath: string) => boolean))[];
 
       /**
        * Deprecated in favor of `allowlist`.
@@ -952,15 +956,19 @@ declare namespace tracer {
        * @deprecated
        * @hidden
        */
-      whitelist?: string | RegExp | ((url: string) => boolean) | (string | RegExp | ((url: string) => boolean))[];
+      whitelist?: string | RegExp | ((urlOrPath: string) => boolean) | (string | RegExp | ((urlOrPath: string) => boolean))[];
 
       /**
-       * List of URLs that should not be instrumented. Takes precedence over
+       * List of URLs/paths that should not be instrumented. Takes precedence over
        * allowlist if a URL matches an entry in both.
        *
+       * Note that when used for an http client the entry represents a full
+       * outbound URL (`https://example.org/api/foo`) but when used as a
+       * server the entry represents an inbound path (`/api/foo`).
+       *
        * @default []
        */
-      blocklist?: string | RegExp | ((url: string) => boolean) | (string | RegExp | ((url: string) => boolean))[];
+      blocklist?: string | RegExp | ((urlOrPath: string) => boolean) | (string | RegExp | ((urlOrPath: string) => boolean))[];
 
       /**
        * Deprecated in favor of `blocklist`.
@@ -968,7 +976,7 @@ declare namespace tracer {
        * @deprecated
        * @hidden
        */
-      blacklist?: string | RegExp | ((url: string) => boolean) | (string | RegExp | ((url: string) => boolean))[];
+      blacklist?: string | RegExp | ((urlOrPath: string) => boolean) | (string | RegExp | ((urlOrPath: string) => boolean))[];
 
       /**
        * An array of headers to include in the span metadata.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "5.11.0",
+  "version": "5.13.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -71,8 +71,8 @@
   },
   "dependencies": {
     "@datadog/native-appsec": "7.1.1",
-    "@datadog/native-iast-rewriter": "2.3.0",
-    "@datadog/native-iast-taint-tracking": "1.7.0",
+    "@datadog/native-iast-rewriter": "2.3.1",
+    "@datadog/native-iast-taint-tracking": "2.1.0",
     "@datadog/native-metrics": "^2.0.0",
     "@datadog/pprof": "5.2.0",
     "@datadog/sketches-js": "^2.1.0",
@@ -81,7 +81,7 @@
     "crypto-randomuuid": "^1.0.0",
     "dc-polyfill": "^0.1.4",
     "ignore": "^5.2.4",
-    "import-in-the-middle": "^1.7.3",
+    "import-in-the-middle": "^1.7.4",
     "int64-buffer": "^0.1.9",
     "ipaddr.js": "^2.1.0",
     "istanbul-lib-coverage": "3.2.0",

package/packages/datadog-esbuild/index.js

@@ -75,7 +75,10 @@ module.exports.setup = function (build) {
     try {
       fullPathToModule = dotFriendlyResolve(args.path, args.resolveDir)
     } catch (err) {
-      console.warn(`MISSING: Unable to find "${args.path}". Is the package dead code?`)
+      if (DEBUG) {
+        console.warn(`Warning: Unable to find "${args.path}".` +
+          "Unless it's dead code this could cause a problem at runtime.")
+      }
       return
     }
     const extracted = extractPackageAndModulePath(fullPathToModule)
@@ -93,7 +96,10 @@ module.exports.setup = function (build) {
       } catch (err) {
         if (err.code === 'MODULE_NOT_FOUND') {
           if (!internal) {
-            console.warn(`MISSING: Unable to find "${extracted.pkg}/package.json". Is the package dead code?`)
+            if (DEBUG) {
+              console.warn(`Warning: Unable to find "${extracted.pkg}/package.json".` +
+              "Unless it's dead code this could cause a problem at runtime.")
+            }
           }
           return
         } else {

package/packages/datadog-instrumentations/src/aws-sdk.js

@@ -162,8 +162,11 @@ function getChannelSuffix (name) {
     'lambda',
     'redshift',
     's3',
+    'sfn',
     'sns',
-    'sqs'
+    'sqs',
+    'states',
+    'stepfunctions'
   ].includes(name)
     ? name
     : 'default'

package/packages/datadog-instrumentations/src/cucumber.js

@@ -22,6 +22,8 @@ const skippableSuitesCh = channel('ci:cucumber:test-suite:skippable')
 const sessionStartCh = channel('ci:cucumber:session:start')
 const sessionFinishCh = channel('ci:cucumber:session:finish')
 
+const workerReportTraceCh = channel('ci:cucumber:worker-report:trace')
+
 const itrSkippedSuitesCh = channel('ci:cucumber:itr:skipped-suites')
 
 const {
@@ -29,7 +31,8 @@ const {
   resetCoverage,
   mergeCoverage,
   fromCoverageMapToCoverage,
-  getTestSuitePath
+  getTestSuitePath,
+  CUCUMBER_WORKER_TRACE_PAYLOAD_CODE
 } = require('../../dd-trace/src/plugins/util/test')
 
 const isMarkedAsUnskippable = (pickle) => {
@@ -47,13 +50,19 @@ const numRetriesByPickleId = new Map()
 
 let pickleByFile = {}
 const pickleResultByFile = {}
+
+const sessionAsyncResource = new AsyncResource('bound-anonymous-fn')
+
 let skippableSuites = []
 let itrCorrelationId = ''
 let isForcedToRun = false
 let isUnskippable = false
+let isSuitesSkippingEnabled = false
 let isEarlyFlakeDetectionEnabled = false
 let earlyFlakeDetectionNumRetries = 0
 let knownTests = []
+let skippedSuites = []
+let isSuitesSkipped = false
 
 function getSuiteStatusFromTestStatuses (testStatuses) {
   if (testStatuses.some(status => status === 'fail')) {
@@ -106,6 +115,43 @@ function getTestStatusFromRetries (testStatuses) {
   return 'pass'
 }
 
+function getChannelPromise (channelToPublishTo) {
+  return new Promise(resolve => {
+    sessionAsyncResource.runInAsyncScope(() => {
+      channelToPublishTo.publish({ onDone: resolve })
+    })
+  })
+}
+
+function getFilteredPickles (runtime, suitesToSkip) {
+  return runtime.pickleIds.reduce((acc, pickleId) => {
+    const test = runtime.eventDataCollector.getPickle(pickleId)
+    const testSuitePath = getTestSuitePath(test.uri, process.cwd())
+
+    const isUnskippable = isMarkedAsUnskippable(test)
+    const isSkipped = suitesToSkip.includes(testSuitePath)
+
+    if (isSkipped && !isUnskippable) {
+      acc.skippedSuites.add(testSuitePath)
+    } else {
+      acc.picklesToRun.push(pickleId)
+    }
+    return acc
+  }, { skippedSuites: new Set(), picklesToRun: [] })
+}
+
+function getPickleByFile (runtime) {
+  return runtime.pickleIds.reduce((acc, pickleId) => {
+    const test = runtime.eventDataCollector.getPickle(pickleId)
+    if (acc[test.uri]) {
+      acc[test.uri].push(test)
+    } else {
+      acc[test.uri] = [test]
+    }
+    return acc
+  }, {})
+}
+
 function wrapRun (pl, isLatestVersion) {
   if (patched.has(pl)) return
 
@@ -125,7 +171,8 @@ function wrapRun (pl, isLatestVersion) {
       testStartCh.publish({
         testName: this.pickle.name,
         testFileAbsolutePath,
-        testSourceLine
+        testSourceLine,
+        isParallel: !!process.env.CUCUMBER_WORKER_ID
       })
       try {
         const promise = run.apply(this, arguments)
@@ -193,12 +240,6 @@ function wrapRun (pl, isLatestVersion) {
 }
 
 function pickleHook (PickleRunner) {
-  if (process.env.CUCUMBER_WORKER_ID) {
-    // Parallel mode is not supported
-    log.warn('Unable to initialize CI Visibility because Cucumber is running in parallel mode.')
-    return PickleRunner
-  }
-
   const pl = PickleRunner.default
 
   wrapRun(pl, false)
@@ -207,12 +248,6 @@ function pickleHook (PickleRunner) {
 }
 
 function testCaseHook (TestCaseRunner) {
-  if (process.env.CUCUMBER_WORKER_ID) {
-    // Parallel mode is not supported
-    log.warn('Unable to initialize CI Visibility because Cucumber is running in parallel mode.')
-    return TestCaseRunner
-  }
-
   const pl = TestCaseRunner.default
 
   wrapRun(pl, true)
@@ -220,76 +255,21 @@ function testCaseHook (TestCaseRunner) {
   return TestCaseRunner
 }
 
-addHook({
-  name: '@cucumber/cucumber',
-  versions: ['7.0.0 - 7.2.1'],
-  file: 'lib/runtime/pickle_runner.js'
-}, pickleHook)
-
-addHook({
-  name: '@cucumber/cucumber',
-  versions: ['>=7.3.0'],
-  file: 'lib/runtime/test_case_runner.js'
-}, testCaseHook)
-
-function getFilteredPickles (runtime, suitesToSkip) {
-  return runtime.pickleIds.reduce((acc, pickleId) => {
-    const test = runtime.eventDataCollector.getPickle(pickleId)
-    const testSuitePath = getTestSuitePath(test.uri, process.cwd())
-
-    const isUnskippable = isMarkedAsUnskippable(test)
-    const isSkipped = suitesToSkip.includes(testSuitePath)
-
-    if (isSkipped && !isUnskippable) {
-      acc.skippedSuites.add(testSuitePath)
-    } else {
-      acc.picklesToRun.push(pickleId)
-    }
-    return acc
-  }, { skippedSuites: new Set(), picklesToRun: [] })
-}
-
-function getPickleByFile (runtime) {
-  return runtime.pickleIds.reduce((acc, pickleId) => {
-    const test = runtime.eventDataCollector.getPickle(pickleId)
-    if (acc[test.uri]) {
-      acc[test.uri].push(test)
-    } else {
-      acc[test.uri] = [test]
-    }
-    return acc
-  }, {})
-}
-
-function getWrappedStart (start, frameworkVersion) {
+function getWrappedStart (start, frameworkVersion, isParallel = false) {
   return async function () {
     if (!libraryConfigurationCh.hasSubscribers) {
       return start.apply(this, arguments)
     }
-    const asyncResource = new AsyncResource('bound-anonymous-fn')
-    let onDone
-
-    const configPromise = new Promise(resolve => {
-      onDone = resolve
-    })
-
-    asyncResource.runInAsyncScope(() => {
-      libraryConfigurationCh.publish({ onDone })
-    })
+    let errorSkippableRequest
 
-    const configurationResponse = await configPromise
+    const configurationResponse = await getChannelPromise(libraryConfigurationCh)
 
     isEarlyFlakeDetectionEnabled = configurationResponse.libraryConfig?.isEarlyFlakeDetectionEnabled
     earlyFlakeDetectionNumRetries = configurationResponse.libraryConfig?.earlyFlakeDetectionNumRetries
+    isSuitesSkippingEnabled = configurationResponse.libraryConfig?.isSuitesSkippingEnabled
 
     if (isEarlyFlakeDetectionEnabled) {
-      const knownTestsPromise = new Promise(resolve => {
-        onDone = resolve
-      })
-      asyncResource.runInAsyncScope(() => {
-        knownTestsCh.publish({ onDone })
-      })
-      const knownTestsResponse = await knownTestsPromise
+      const knownTestsResponse = await getChannelPromise(knownTestsCh)
       if (!knownTestsResponse.err) {
         knownTests = knownTestsResponse.knownTests
       } else {
@@ -297,35 +277,26 @@ function getWrappedStart (start, frameworkVersion) {
       }
     }
 
-    const skippableSuitesPromise = new Promise(resolve => {
-      onDone = resolve
-    })
+    if (isSuitesSkippingEnabled) {
+      const skippableResponse = await getChannelPromise(skippableSuitesCh)
 
-    asyncResource.runInAsyncScope(() => {
-      skippableSuitesCh.publish({ onDone })
-    })
+      errorSkippableRequest = skippableResponse.err
+      skippableSuites = skippableResponse.skippableSuites
 
-    const skippableResponse = await skippableSuitesPromise
+      if (!errorSkippableRequest) {
+        const filteredPickles = getFilteredPickles(this, skippableSuites)
+        const { picklesToRun } = filteredPickles
+        isSuitesSkipped = picklesToRun.length !== this.pickleIds.length
 
-    const err = skippableResponse.err
-    skippableSuites = skippableResponse.skippableSuites
+        log.debug(
+          () => `${picklesToRun.length} out of ${this.pickleIds.length} suites are going to run.`
+        )
 
-    let skippedSuites = []
-    let isSuitesSkipped = false
+        this.pickleIds = picklesToRun
 
-    if (!err) {
-      const filteredPickles = getFilteredPickles(this, skippableSuites)
-      const { picklesToRun } = filteredPickles
-      isSuitesSkipped = picklesToRun.length !== this.pickleIds.length
-
-      log.debug(
-        () => `${picklesToRun.length} out of ${this.pickleIds.length} suites are going to run.`
-      )
-
-      this.pickleIds = picklesToRun
-
-      skippedSuites = Array.from(filteredPickles.skippedSuites)
-      itrCorrelationId = skippableResponse.itrCorrelationId
+        skippedSuites = Array.from(filteredPickles.skippedSuites)
+        itrCorrelationId = skippableResponse.itrCorrelationId
+      }
     }
 
     pickleByFile = getPickleByFile(this)
@@ -333,11 +304,11 @@ function getWrappedStart (start, frameworkVersion) {
     const processArgv = process.argv.slice(2).join(' ')
     const command = process.env.npm_lifecycle_script || `cucumber-js ${processArgv}`
 
-    asyncResource.runInAsyncScope(() => {
+    sessionAsyncResource.runInAsyncScope(() => {
       sessionStartCh.publish({ command, frameworkVersion })
     })
 
-    if (!err && skippedSuites.length) {
+    if (!errorSkippableRequest && skippedSuites.length) {
       itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion })
     }
 
@@ -355,7 +326,7 @@ function getWrappedStart (start, frameworkVersion) {
       global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap)
     }
 
-    asyncResource.runInAsyncScope(() => {
+    sessionAsyncResource.runInAsyncScope(() => {
       sessionFinishCh.publish({
         status: success ? 'pass' : 'fail',
         isSuitesSkipped,
@@ -363,7 +334,8 @@ function getWrappedStart (start, frameworkVersion) {
         numSkippedSuites: skippedSuites.length,
         hasUnskippableSuites: isUnskippable,
         hasForcedToRunSuites: isForcedToRun,
-        isEarlyFlakeDetectionEnabled
+        isEarlyFlakeDetectionEnabled,
+        isParallel
       })
     })
     return success
@@ -432,7 +404,8 @@ function getWrappedRunTest (runTestFunction) {
 
         testSuiteCodeCoverageCh.publish({
           coverageFiles,
-          suiteFile: testFileAbsolutePath
+          suiteFile: testFileAbsolutePath,
+          testSuitePath
         })
         // We need to reset coverage to get a code coverage per suite
         // Before that, we preserve the original coverage
@@ -440,14 +413,97 @@ function getWrappedRunTest (runTestFunction) {
         resetCoverage(global.__coverage__)
       }
 
-      testSuiteFinishCh.publish(testSuiteStatus)
+      testSuiteFinishCh.publish({ status: testSuiteStatus, testSuitePath })
     }
 
     return runTestCaseResult
   }
 }
 
-// From 7.3.0 onwards, runPickle becomes runTestCase
+function getWrappedParseWorkerMessage (parseWorkerMessageFunction) {
+  return function (worker, message) {
+    // If the message is an array, it's a dd-trace message, so we need to stop cucumber processing,
+    // or cucumber will throw an error
+    // TODO: identify the message better
+    if (Array.isArray(message)) {
+      const [messageCode, payload] = message
+      if (messageCode === CUCUMBER_WORKER_TRACE_PAYLOAD_CODE) {
+        sessionAsyncResource.runInAsyncScope(() => {
+          workerReportTraceCh.publish(payload)
+        })
+        return
+      }
+    }
+
+    const { jsonEnvelope } = message
+    if (!jsonEnvelope) {
+      return parseWorkerMessageFunction.apply(this, arguments)
+    }
+    let parsed = jsonEnvelope
+
+    if (typeof parsed === 'string') {
+      try {
+        parsed = JSON.parse(jsonEnvelope)
+      } catch (e) {
+        // ignore errors and continue
+        return parseWorkerMessageFunction.apply(this, arguments)
+      }
+    }
+    if (parsed.testCaseStarted) {
+      const { pickleId } = this.eventDataCollector.testCaseMap[parsed.testCaseStarted.testCaseId]
+      const pickle = this.eventDataCollector.getPickle(pickleId)
+      const testFileAbsolutePath = pickle.uri
+      // First test in suite
+      if (!pickleResultByFile[testFileAbsolutePath]) {
+        pickleResultByFile[testFileAbsolutePath] = []
+        testSuiteStartCh.publish({
+          testSuitePath: getTestSuitePath(testFileAbsolutePath, process.cwd())
+        })
+      }
+    }
+
+    const parseWorkerResponse = parseWorkerMessageFunction.apply(this, arguments)
+
+    // after calling `parseWorkerMessageFunction`, the test status can already be read
+    if (parsed.testCaseFinished) {
+      const { pickle, worstTestStepResult } =
+        this.eventDataCollector.getTestCaseAttempt(parsed.testCaseFinished.testCaseStartedId)
+
+      const { status } = getStatusFromResultLatest(worstTestStepResult)
+
+      const testFileAbsolutePath = pickle.uri
+      const finished = pickleResultByFile[testFileAbsolutePath]
+      finished.push(status)
+
+      if (finished.length === pickleByFile[testFileAbsolutePath].length) {
+        testSuiteFinishCh.publish({
+          status: getSuiteStatusFromTestStatuses(finished),
+          testSuitePath: getTestSuitePath(testFileAbsolutePath, process.cwd())
+        })
+      }
+    }
+
+    return parseWorkerResponse
+  }
+}
+
+// Test start / finish for older versions. The only hook executed in workers when in parallel mode
+addHook({
+  name: '@cucumber/cucumber',
+  versions: ['7.0.0 - 7.2.1'],
+  file: 'lib/runtime/pickle_runner.js'
+}, pickleHook)
+
+// Test start / finish for newer versions. The only hook executed in workers when in parallel mode
+addHook({
+  name: '@cucumber/cucumber',
+  versions: ['>=7.3.0'],
+  file: 'lib/runtime/test_case_runner.js'
+}, testCaseHook)
+
+// From 7.3.0 onwards, runPickle becomes runTestCase. Not executed in parallel mode.
+// `getWrappedStart` generates session start and finish events
+// `getWrappedRunTest` generates suite start and finish events
 addHook({
   name: '@cucumber/cucumber',
   versions: ['>=7.3.0'],
@@ -459,6 +515,9 @@ addHook({
   return runtimePackage
 })
 
+// Not executed in parallel mode.
+// `getWrappedStart` generates session start and finish events
+// `getWrappedRunTest` generates suite start and finish events
 addHook({
   name: '@cucumber/cucumber',
   versions: ['>=7.0.0 <7.3.0'],
@@ -469,3 +528,21 @@ addHook({
 
   return runtimePackage
 })
+
+// Only executed in parallel mode.
+// `getWrappedStart` generates session start and finish events
+// `getWrappedGiveWork` generates suite start events and sets pickleResultByFile (used by suite finish events)
+// `getWrappedParseWorkerMessage` generates suite finish events
+addHook({
+  name: '@cucumber/cucumber',
+  versions: ['>=8.0.0'],
+  file: 'lib/runtime/parallel/coordinator.js'
+}, (coordinatorPackage, frameworkVersion) => {
+  shimmer.wrap(coordinatorPackage.default.prototype, 'start', start => getWrappedStart(start, frameworkVersion, true))
+  shimmer.wrap(
+    coordinatorPackage.default.prototype,
+    'parseWorkerMessage',
+    parseWorkerMessage => getWrappedParseWorkerMessage(parseWorkerMessage)
+  )
+  return coordinatorPackage
+})

package/packages/datadog-instrumentations/src/helpers/fetch.js

@@ -7,11 +7,16 @@ exports.createWrapFetch = function createWrapFetch (Request, ch) {
     return function (input, init) {
       if (!ch.start.hasSubscribers) return fetch.apply(this, arguments)
 
-      const req = new Request(input, init)
-      const headers = req.headers
-      const ctx = { req, headers }
+      if (input instanceof Request) {
+        const ctx = { req: input }
 
-      return ch.tracePromise(() => fetch.call(this, req, { headers: ctx.headers }), ctx)
+        return ch.tracePromise(() => fetch.call(this, input, init), ctx)
+      } else {
+        const req = new Request(input, init)
+        const ctx = { req }
+
+        return ch.tracePromise(() => fetch.call(this, req), ctx)
+      }
     }
   }
 }

package/packages/datadog-instrumentations/src/helpers/hooks.js

@@ -66,6 +66,7 @@ module.exports = {
   kafkajs: () => require('../kafkajs'),
   ldapjs: () => require('../ldapjs'),
   'limitd-client': () => require('../limitd-client'),
+  lodash: () => require('../lodash'),
   mariadb: () => require('../mariadb'),
   memcached: () => require('../memcached'),
   'microgateway-core': () => require('../microgateway-core'),

package/packages/datadog-instrumentations/src/lodash.js

@@ -0,0 +1,31 @@
+'use strict'
+
+const { channel, addHook } = require('./helpers/instrument')
+
+const shimmer = require('../../datadog-shimmer')
+
+addHook({ name: 'lodash', versions: ['>=4'] }, lodash => {
+  const lodashOperationCh = channel('datadog:lodash:operation')
+
+  const instrumentedLodashFn = ['trim', 'trimStart', 'trimEnd', 'toLower', 'toUpper', 'join']
+
+  shimmer.massWrap(
+    lodash,
+    instrumentedLodashFn,
+    lodashFn => {
+      return function () {
+        if (!lodashOperationCh.hasSubscribers) {
+          return lodashFn.apply(this, arguments)
+        }
+
+        const result = lodashFn.apply(this, arguments)
+        const message = { operation: lodashFn.name, arguments, result }
+        lodashOperationCh.publish(message)
+
+        return message.result
+      }
+    }
+  )
+
+  return lodash
+})