dd-trace 3.0.0 → 3.1.0

package/LICENSE-3rdparty.csv

@@ -7,6 +7,7 @@ require,crypto-randomuuid,MIT,Copyright 2021 Node.js Foundation and contributors
 require,diagnostics_channel,MIT,Copyright 2021 Simon D.
 require,ignore,MIT,Copyright 2013 Kael Zhang and contributors
 require,import-in-the-middle,Apache license 2.0,Copyright 2021 Datadog Inc.
+require,istanbul-lib-coverage,BSD-3-Clause,Copyright 2012-2015 Yahoo! Inc.
 require,koalas,MIT,Copyright 2013-2017 Brian Woodward
 require,limiter,MIT,Copyright 2011 John Hurliman
 require,lodash.kebabcase,MIT,Copyright JS Foundation and other contributors

package/MIGRATING.md

@@ -108,20 +108,20 @@ respectively.
 
 The use of `'dd-trace/ci/jest/env'` in [`testEnvironment`](https://jestjs.io/docs/configuration#testenvironment-string)
 is no longer supported. 
-The way to instrument your `jest` tests now is by passing the `NODE_OPTIONS='-r dd-trace/ci/init'`
-environment variable to the process running the tests.
+To instrument `jest` tests now, add `'-r dd-trace/ci/init'` to the `NODE_OPTIONS` environment
+variable passed to the process running the tests, for example, `NODE_OPTIONS='-r dd-trace/ci/init' yarn test`.
 
 #### Mocha
 
 The use of `--require dd-trace/ci/init` as a `mocha` flag is no longer supported. 
-The way to instrument your `mocha` tests now is by passing the `NODE_OPTIONS='-r dd-trace/ci/init'`
-environment variable to the process running the tests.
+To instrument `mocha` tests now, add `'-r dd-trace/ci/init'` to the `NODE_OPTIONS` environment
+variable passed to the process running the tests, for example, `NODE_OPTIONS='-r dd-trace/ci/init' yarn test`.
 
 #### Cucumber
 
 The use of `--require-module dd-trace/ci/init` as a `cucumber-js` flag is no longer supported.
-The way to instrument your `cucumber-js` tests now is by passing the `NODE_OPTIONS='-r dd-trace/ci/init'`
-environment variable to the process running the tests.
+To instrument `cucumber-js` tests now, add `'-r dd-trace/ci/init'` to the `NODE_OPTIONS` environment
+variable passed to the process running the tests, for example, `NODE_OPTIONS='-r dd-trace/ci/init' yarn test`.
 
 ## 1.0 to 2.0
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -66,6 +66,7 @@
     "diagnostics_channel": "^1.1.0",
     "ignore": "^5.2.0",
     "import-in-the-middle": "^1.3.0",
+    "istanbul-lib-coverage": "3.2.0",
     "koalas": "^1.0.2",
     "limiter": "^1.1.4",
     "lodash.kebabcase": "^4.1.1",

package/packages/datadog-core/src/storage/async_resource.js

@@ -6,9 +6,27 @@ const { channel } = require('diagnostics_channel')
 const beforeCh = channel('dd-trace:storage:before')
 const afterCh = channel('dd-trace:storage:after')
 
+let PrivateSymbol = Symbol
+function makePrivateSymbol () {
+  // eslint-disable-next-line no-new-func
+  PrivateSymbol = new Function('name', 'return %CreatePrivateSymbol(name)')
+}
+
+try {
+  makePrivateSymbol()
+} catch (e) {
+  try {
+    const v8 = require('v8')
+    v8.setFlagsFromString('--allow-natives-syntax')
+    makePrivateSymbol()
+    v8.setFlagsFromString('--no-allow-natives-syntax')
+  // eslint-disable-next-line no-empty
+  } catch (e) {}
+}
+
 class AsyncResourceStorage {
   constructor () {
-    this._ddResourceStore = Symbol('ddResourceStore')
+    this._ddResourceStore = PrivateSymbol('ddResourceStore')
     this._enabled = false
     this._hook = createHook(this._createHook())
   }

package/packages/datadog-instrumentations/src/cucumber.js

@@ -7,6 +7,7 @@ const runStartCh = channel('ci:cucumber:run:start')
 const runFinishCh = channel('ci:cucumber:run:finish')
 const runStepStartCh = channel('ci:cucumber:run-step:start')
 const errorCh = channel('ci:cucumber:error')
+const sessionFinishCh = channel('ci:cucumber:session:finish')
 
 // TODO: remove in a later major version
 const patched = new WeakSet()
@@ -128,4 +129,18 @@ addHook({
   file: 'lib/runtime/test_case_runner.js'
 }, testCaseHook)
 
+addHook({
+  name: '@cucumber/cucumber',
+  versions: ['>=7.0.0'],
+  file: 'lib/runtime/index.js'
+}, (Runtime) => {
+  shimmer.wrap(Runtime.default.prototype, 'start', start => async function () {
+    const result = await start.apply(this, arguments)
+    sessionFinishCh.publish(undefined)
+    return result
+  })
+
+  return Runtime
+})
+
 module.exports = { pickleHook, testCaseHook }

package/packages/datadog-instrumentations/src/helpers/instrumentations.js

@@ -1,3 +1,7 @@
 'use strict'
 
-module.exports = {}
+const sym = Symbol.for('_ddtrace_instrumentations')
+
+global[sym] = global[sym] || {}
+
+module.exports = global[sym]

package/packages/datadog-instrumentations/src/jest.js

@@ -1,5 +1,5 @@
 'use strict'
-
+const istanbul = require('istanbul-lib-coverage')
 const { addHook, channel, AsyncResource } = require('./helpers/instrument')
 const shimmer = require('../../datadog-shimmer')
 
@@ -7,7 +7,8 @@ const testStartCh = channel('ci:jest:test:start')
 const testSkippedCh = channel('ci:jest:test:skip')
 const testRunFinishCh = channel('ci:jest:test:finish')
 const testErrCh = channel('ci:jest:test:err')
-const testSuiteFinish = channel('ci:jest:test-suite:finish')
+
+const testCodeCoverageCh = channel('ci:jest:test:code-coverage')
 
 const {
   getTestSuitePath,
@@ -16,6 +17,24 @@ const {
 
 const { getFormattedJestTestParameters, getJestTestName } = require('../../datadog-plugin-jest/src/util')
 
+// This function also resets the coverage counters
+function extractCoverageInformation (coverage, rootDir) {
+  const coverageMap = istanbul.createCoverageMap(coverage)
+
+  return coverageMap
+    .files()
+    .filter(filename => {
+      const fileCoverage = coverageMap.fileCoverageFor(filename)
+      const lineCoverage = fileCoverage.getLineCoverage()
+      const isAnyLineExecuted = Object.entries(lineCoverage).some(([, numExecutions]) => !!numExecutions)
+
+      fileCoverage.resetHits()
+
+      return isAnyLineExecuted
+    })
+    .map(filename => filename.replace(`${rootDir}/`, ''))
+}
+
 const specStatusToTestStatus = {
   'pending': 'skip',
   'disabled': 'skip',
@@ -49,15 +68,11 @@ function getWrappedEnvironment (BaseEnvironment) {
     constructor (config, context) {
       super(config, context)
       const rootDir = config.globalConfig ? config.globalConfig.rootDir : config.rootDir
+      this.rootDir = rootDir
       this.testSuite = getTestSuitePath(context.testPath, rootDir)
       this.nameToParams = {}
       this.global._ddtrace = global._ddtrace
     }
-    async teardown () {
-      super.teardown().finally(() => {
-        testSuiteFinish.publish()
-      })
-    }
 
     async handleTestEvent (event, state) {
       if (super.handleTestEvent) {
@@ -99,6 +114,10 @@ function getWrappedEnvironment (BaseEnvironment) {
       if (event.name === 'test_done') {
         const asyncResource = asyncResources.get(event.test)
         asyncResource.runInAsyncScope(() => {
+          if (this.global.__coverage__) {
+            const coverageFiles = extractCoverageInformation(this.global.__coverage__, this.rootDir)
+            testCodeCoverageCh.publish(coverageFiles)
+          }
           let status = 'pass'
           if (event.test.errors && event.test.errors.length) {
             status = 'fail'
@@ -111,10 +130,13 @@ function getWrappedEnvironment (BaseEnvironment) {
         })
       }
       if (event.name === 'test_skip' || event.name === 'test_todo') {
-        testSkippedCh.publish({
-          name: getJestTestName(event.test),
-          suite: this.testSuite,
-          runner: 'jest-circus'
+        const asyncResource = new AsyncResource('bound-anonymous-fn')
+        asyncResource.runInAsyncScope(() => {
+          testSkippedCh.publish({
+            name: getJestTestName(event.test),
+            suite: this.testSuite,
+            runner: 'jest-circus'
+          })
         })
       }
     }

package/packages/datadog-plugin-cucumber/src/index.js

@@ -30,6 +30,10 @@ class CucumberPlugin extends Plugin {
     const sourceRoot = process.cwd()
     const codeOwnersEntries = getCodeOwnersFileEntries(sourceRoot)
 
+    this.addSub('ci:cucumber:session:finish', () => {
+      this.tracer._exporter._writer.flush()
+    })
+
     this.addSub('ci:cucumber:run:start', ({ pickleName, pickleUri }) => {
       const store = storage.getStore()
       const childOf = store ? store.span : store

package/packages/datadog-plugin-jest/src/index.js

@@ -15,6 +15,9 @@ const {
   TEST_CODE_OWNERS
 } = require('../../dd-trace/src/plugins/util/test')
 
+// https://github.com/facebook/jest/blob/d6ad15b0f88a05816c2fe034dd6900d28315d570/packages/jest-worker/src/types.ts#L38
+const CHILD_MESSAGE_END = 2
+
 function getTestSpanMetadata (tracer, test) {
   const childOf = getTestParentSpan(tracer)
 
@@ -38,9 +41,31 @@ class JestPlugin extends Plugin {
   constructor (...args) {
     super(...args)
 
+    // Used to handle the end of a jest worker to be able to flush
+    const handler = ([message]) => {
+      if (message === CHILD_MESSAGE_END) {
+        this.tracer._exporter._writer.flush(() => {
+          // eslint-disable-next-line
+          // https://github.com/facebook/jest/blob/24ed3b5ecb419c023ee6fdbc838f07cc028fc007/packages/jest-worker/src/workers/processChild.ts#L118-L133
+          // Only after the flush is done we clean up open handles
+          // so the worker process can hopefully exit gracefully
+          process.removeListener('message', handler)
+        })
+      }
+    }
+    process.on('message', handler)
+
     this.testEnvironmentMetadata = getTestEnvironmentMetadata('jest', this.config)
     this.codeOwnersEntries = getCodeOwnersFileEntries()
 
+    this.addSub('ci:jest:test:code-coverage', (coverageFiles) => {
+      if (!this.config.isAgentlessEnabled || !this.config.isIntelligentTestRunnerEnabled) {
+        return
+      }
+      const testSpan = storage.getStore().span
+      this.tracer._exporter.exportCoverage({ testSpan, coverageFiles })
+    })
+
     this.addSub('ci:jest:test:start', (test) => {
       const store = storage.getStore()
       const span = this.startTestSpan(test)
@@ -55,10 +80,6 @@ class JestPlugin extends Plugin {
       finishAllTraceSpans(span)
     })
 
-    this.addSub('ci:jest:test-suite:finish', () => {
-      this.tracer._exporter._writer.flush()
-    })
-
     this.addSub('ci:jest:test:err', (error) => {
       if (error) {
         const span = storage.getStore().span

package/packages/datadog-plugin-mongodb-core/src/index.js

@@ -14,7 +14,7 @@ class MongodbCorePlugin extends Plugin {
 
     this.addSub(`apm:mongodb:query:start`, ({ ns, ops, options, name }) => {
       const query = getQuery(ops)
-      const resource = getResource(ns, query, name)
+      const resource = truncate(getResource(ns, query, name))
       const store = storage.getStore()
       const childOf = store ? store.span : store
       const span = this.tracer.startSpan('mongodb.query', {
@@ -55,8 +55,8 @@ class MongodbCorePlugin extends Plugin {
 
 function getQuery (cmd) {
   if (!cmd || typeof cmd !== 'object' || Array.isArray(cmd)) return
-  if (cmd.query) return JSON.stringify(sanitize(cmd.query))
-  if (cmd.filter) return JSON.stringify(sanitize(cmd.filter))
+  if (cmd.query) return JSON.stringify(limitDepth(cmd.query))
+  if (cmd.filter) return JSON.stringify(limitDepth(cmd.filter))
 }
 
 function getResource (ns, query, operationName) {
@@ -69,12 +69,25 @@ function getResource (ns, query, operationName) {
   return parts.join(' ')
 }
 
+function truncate (input) {
+  return input.slice(0, Math.min(input.length, 10000))
+}
+
+function simplify (input) {
+  return isBSON(input) ? input.toHexString() : input
+}
+
+function shouldSimplify (input) {
+  return !isObject(input) || isBSON(input)
+}
+
 function shouldHide (input) {
-  return !isObject(input) || Buffer.isBuffer(input) || isBSON(input)
+  return Buffer.isBuffer(input) || typeof input === 'function'
 }
 
-function sanitize (input) {
+function limitDepth (input) {
   if (shouldHide(input)) return '?'
+  if (shouldSimplify(input)) return simplify(input)
 
   const output = {}
   const queue = [{
@@ -92,8 +105,10 @@ function sanitize (input) {
       if (typeof input[key] === 'function') continue
 
       const child = input[key]
-      if (depth >= 20 || shouldHide(child)) {
+      if (depth >= 10 || shouldHide(child)) {
         output[key] = '?'
+      } else if (shouldSimplify(child)) {
+        output[key] = simplify(child)
       } else {
         queue.push({
           input: child,

package/packages/datadog-plugin-oracledb/src/index.js

@@ -3,6 +3,7 @@
 const Plugin = require('../../dd-trace/src/plugins/plugin')
 const { storage } = require('../../datadog-core')
 const analyticsSampler = require('../../dd-trace/src/analytics_sampler')
+const log = require('../../dd-trace/src/log')
 
 class OracledbPlugin extends Plugin {
   static get name () {
@@ -14,18 +15,25 @@ class OracledbPlugin extends Plugin {
 
     this.addSub('apm:oracledb:execute:start', ({ query, connAttrs }) => {
       const service = getServiceName(this.tracer, this.config, connAttrs)
-      const connectStringObj = new URL('http://' + connAttrs.connectString)
+      let connectStringObj
+      try {
+        connectStringObj = new URL(`http://${connAttrs.connectString}`)
+      } catch (e) {
+        log.error(e)
+      }
       const tags = {
         'span.kind': 'client',
         'span.type': 'sql',
         'sql.query': query,
-        'db.instance': connectStringObj.pathname.substring(1),
-        'db.hostname': connectStringObj.hostname,
         'db.user': this.config.user,
-        'db.port': connectStringObj.port,
         'resource.name': query,
         'service.name': service
       }
+      if (typeof connectStringObj !== 'undefined') {
+        tags['db.instance'] = connectStringObj.pathname.substring(1)
+        tags['db.hostname'] = connectStringObj.hostname
+        tags['db.port'] = connectStringObj.port
+      }
       const store = storage.getStore()
       const childOf = store ? store.span : store
       const span = this.tracer.startSpan('oracle.query', {

package/packages/dd-trace/index.js

@@ -1,7 +1,7 @@
 'use strict'
 
 if (!global._ddtrace) {
-  const TracerProxy = require('./src/proxy')
+  const TracerProxy = require('./src')
 
   Object.defineProperty(global, '_ddtrace', {
     value: new TracerProxy(),

package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js

@@ -0,0 +1,50 @@
+'use strict'
+const request = require('../../../exporters/common/request')
+const log = require('../../../log')
+
+const { CoverageCIVisibilityEncoder } = require('../../../encode/coverage-ci-visibility')
+const BaseWriter = require('../../../exporters/common/writer')
+
+function safeJSONStringify (value) {
+  return JSON.stringify(value, (key, value) =>
+    key !== 'dd-api-key' ? value : undefined
+  )
+}
+
+class Writer extends BaseWriter {
+  constructor ({ url }) {
+    super(...arguments)
+    this._url = url
+    this._encoder = new CoverageCIVisibilityEncoder()
+  }
+
+  _sendPayload (form, _, done) {
+    const options = {
+      path: '/api/v2/citestcov',
+      method: 'POST',
+      headers: {
+        'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY,
+        ...form.getHeaders()
+      },
+      timeout: 15000
+    }
+
+    options.protocol = this._url.protocol
+    options.hostname = this._url.hostname
+    options.port = this._url.port
+
+    log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
+
+    request(form, options, (err, res) => {
+      if (err) {
+        log.error(err)
+        done()
+        return
+      }
+      log.debug(`Response from the intake: ${res}`)
+      done()
+    })
+  }
+}
+
+module.exports = Writer

package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js

@@ -2,30 +2,75 @@
 
 const URL = require('url').URL
 const Writer = require('./writer')
-const Scheduler = require('../../../exporters/scheduler')
+const CoverageWriter = require('./coverage-writer')
+
+const log = require('../../../log')
 
 class AgentlessCiVisibilityExporter {
   constructor (config) {
-    const { flushInterval, tags, site, url } = config
+    this._config = config
+    const { tags, site, url, isIntelligentTestRunnerEnabled } = config
+    this._isIntelligentTestRunnerEnabled = isIntelligentTestRunnerEnabled
     this._url = url || new URL(`https://citestcycle-intake.${site}`)
     this._writer = new Writer({ url: this._url, tags })
+    this._timer = undefined
+    this._coverageTimer = undefined
+
+    this._coverageUrl = url || new URL(`https://event-platform-intake.${site}`)
+    this._coverageWriter = new CoverageWriter({ url: this._coverageUrl })
+
+    process.once('beforeExit', () => {
+      this._writer.flush()
+      this._coverageWriter.flush()
+    })
+  }
 
-    if (flushInterval > 0) {
-      this._scheduler = new Scheduler(() => this._writer.flush(), flushInterval)
+  exportCoverage ({ testSpan, coverageFiles }) {
+    const formattedCoverage = {
+      traceId: testSpan.context()._traceId,
+      spanId: testSpan.context()._spanId,
+      files: coverageFiles
+    }
+    this._coverageWriter.append(formattedCoverage)
+
+    const { flushInterval } = this._config
+
+    if (flushInterval === 0) {
+      this._coverageWriter.flush()
+    } else if (flushInterval > 0 && !this._coverageTimer) {
+      this._coverageTimer = setTimeout(() => {
+        this._coverageWriter.flush()
+        this._coverageTimer = clearTimeout(this._coverageTimer)
+      }, flushInterval).unref()
     }
-    this._scheduler && this._scheduler.start()
   }
 
   export (trace) {
     this._writer.append(trace)
 
-    if (!this._scheduler) {
+    const { flushInterval } = this._config
+
+    if (flushInterval === 0) {
       this._writer.flush()
+    } else if (flushInterval > 0 && !this._timer) {
+      this._timer = setTimeout(() => {
+        this._writer.flush()
+        this._timer = clearTimeout(this._timer)
+      }, flushInterval).unref()
     }
   }
 
-  flush () {
-    this._writer.flush()
+  setUrl (url, coverageUrl = url) {
+    try {
+      url = new URL(url)
+      coverageUrl = new URL(coverageUrl)
+      this._url = url
+      this._coverageUrl = coverageUrl
+      this._writer.setUrl(url)
+      this._coverageWriter.setUrl(coverageUrl)
+    } catch (e) {
+      log.error(e)
+    }
   }
 }
 

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -5,16 +5,37 @@ const log = require('../../../log')
 const { AgentlessCiVisibilityEncoder } = require('../../../encode/agentless-ci-visibility')
 const BaseWriter = require('../../../exporters/common/writer')
 
+function safeJSONStringify (value) {
+  return JSON.stringify(value, (key, value) =>
+    key !== 'dd-api-key' ? value : undefined
+  )
+}
+
 class Writer extends BaseWriter {
   constructor ({ url, tags }) {
     super(...arguments)
     const { 'runtime-id': runtimeId, env, service } = tags
     this._url = url
-    this._encoder = new AgentlessCiVisibilityEncoder({ runtimeId, env, service })
+    this._encoder = new AgentlessCiVisibilityEncoder(this, { runtimeId, env, service })
   }
 
   _sendPayload (data, _, done) {
-    makeRequest(data, this._url, (err, res) => {
+    const options = {
+      path: '/api/v2/citestcycle',
+      method: 'POST',
+      headers: {
+        'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY,
+        'Content-Type': 'application/msgpack'
+      },
+      timeout: 15000
+    }
+
+    options.protocol = this._url.protocol
+    options.hostname = this._url.hostname
+    options.port = this._url.port
+
+    log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
+    request(data, options, (err, res) => {
       if (err) {
         log.error(err)
         done()
@@ -26,26 +47,4 @@ class Writer extends BaseWriter {
   }
 }
 
-function makeRequest (data, url, cb) {
-  const options = {
-    path: '/api/v2/citestcycle',
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/msgpack',
-      'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY
-    },
-    timeout: 15000
-  }
-
-  options.protocol = url.protocol
-  options.hostname = url.hostname
-  options.port = url.port
-
-  log.debug(() => `Request to the intake: ${JSON.stringify(options)}`)
-
-  request(data, options, false, (err, res) => {
-    cb(err, res)
-  })
-}
-
 module.exports = Writer

package/packages/dd-trace/src/config.js

@@ -70,6 +70,12 @@ class Config {
       null
     )
     const DD_CIVISIBILITY_AGENTLESS_URL = process.env.DD_CIVISIBILITY_AGENTLESS_URL
+
+    const DD_CIVISIBILITY_ITR_ENABLED = coalesce(
+      process.env.DD_CIVISIBILITY_ITR_ENABLED,
+      false
+    )
+
     const DD_SERVICE = options.service ||
       process.env.DD_SERVICE ||
       process.env.DD_SERVICE_NAME ||
@@ -262,6 +268,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       obfuscatorValueRegex: DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP
     }
     this.isGitUploadEnabled = isTrue(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED)
+    this.isIntelligentTestRunnerEnabled = isTrue(DD_CIVISIBILITY_ITR_ENABLED)
 
     tagger.add(this.tags, {
       service: this.service,