dd-trace 5.0.0-pre-58d2e20 → 5.0.0-pre-8a512a2

package/MIGRATING.md

@@ -29,20 +29,6 @@ switching to `jest-circus` to anyone still using `jest-jasmine2`.
 
 We now support only Next.js 10.2 and up.
 
-### W3C headers are now prioritized over Datadog headers
-
-As we move towards open standards, we have decided to prioritize W3C Trace
-Context headers over our own vendor-specific headers for context propagation
-across services. For most applications this shouldn't change anything and
-distributed tracing should continue to work seamlessly.
-
-In some rare cases it's possible that some of the services involved in a trace
-are not instrumented by Datadog at all which can cause spans within the trace to
-become disconnected. While the data would still be available in the UI, the
-relationship between spans would no longer be visible. This can be addressed by
-restoring the previous behaviour using
-`DD_TRACE_PROPAGATION_STYLE='datadog,tracecontext'`.
-
 ## 2.0 to 3.0
 
 ### Node 12 is no longer supported

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "5.0.0-pre-58d2e20",
+  "version": "5.0.0-pre-8a512a2",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -68,7 +68,7 @@
   "dependencies": {
     "@datadog/native-appsec": "^3.2.0",
     "@datadog/native-iast-rewriter": "2.0.1",
-    "@datadog/native-iast-taint-tracking": "^1.5.0",
+    "@datadog/native-iast-taint-tracking": "1.5.0",
     "@datadog/native-metrics": "^2.0.0",
     "@datadog/pprof": "3.1.0",
     "@datadog/sketches-js": "^2.1.0",
@@ -76,10 +76,10 @@
     "@opentelemetry/core": "^1.14.0",
     "crypto-randomuuid": "^1.0.0",
     "diagnostics_channel": "^1.1.0",
-    "ignore": "^5.2.0",
-    "import-in-the-middle": "^1.3.5",
+    "ignore": "^5.2.4",
+    "import-in-the-middle": "^1.4.1",
     "int64-buffer": "^0.1.9",
-    "ipaddr.js": "^2.0.1",
+    "ipaddr.js": "^2.1.0",
     "istanbul-lib-coverage": "3.2.0",
     "koalas": "^1.0.2",
     "limiter": "^1.1.4",
@@ -91,24 +91,24 @@
     "methods": "^1.1.2",
     "module-details-from-path": "^1.0.3",
     "msgpack-lite": "^0.1.26",
-    "node-abort-controller": "^3.0.1",
+    "node-abort-controller": "^3.1.1",
     "opentracing": ">=0.12.1",
     "path-to-regexp": "^0.1.2",
     "protobufjs": "^7.2.4",
-    "retry": "^0.10.1",
-    "semver": "^7.3.8"
+    "retry": "^0.13.1",
+    "semver": "^7.5.4"
   },
   "devDependencies": {
     "@types/node": ">=16",
     "autocannon": "^4.5.2",
     "axios": "^0.21.2",
     "benchmark": "^2.1.4",
-    "body-parser": "^1.18.2",
-    "chai": "^4.2.0",
-    "chalk": "^3.0.0",
-    "checksum": "^0.1.1",
-    "cli-table3": "^0.5.1",
-    "dotenv": "8.2.0",
+    "body-parser": "^1.20.2",
+    "chai": "^4.3.7",
+    "chalk": "^5.3.0",
+    "checksum": "^1.0.0",
+    "cli-table3": "^0.6.3",
+    "dotenv": "16.3.1",
     "esbuild": "0.16.12",
     "eslint": "^8.23.0",
     "eslint-config-standard": "^11.0.0-beta.0",
@@ -118,13 +118,13 @@
     "eslint-plugin-node": "^5.2.1",
     "eslint-plugin-promise": "^3.6.0",
     "eslint-plugin-standard": "^3.0.1",
-    "express": "^4.16.2",
+    "express": "^4.18.2",
     "get-port": "^3.2.0",
     "glob": "^7.1.6",
     "graphql": "0.13.2",
     "jszip": "^3.5.0",
     "knex": "^2.4.2",
-    "mkdirp": "^0.5.1",
+    "mkdirp": "^3.0.1",
     "mocha": "8",
     "multer": "^1.4.5-lts.1",
     "nock": "^11.3.3",
@@ -132,9 +132,9 @@
     "pprof-format": "^2.0.7",
     "proxyquire": "^1.8.0",
     "rimraf": "^3.0.0",
-    "sinon": "^11.1.2",
+    "sinon": "^15.2.0",
     "sinon-chai": "^3.7.0",
-    "tap": "^16.3.4",
-    "tape": "^4.9.1"
+    "tap": "^16.3.7",
+    "tape": "^5.6.5"
   }
 }

package/packages/datadog-esbuild/index.js

@@ -2,73 +2,105 @@
 
 /* eslint-disable no-console */
 
-const NAMESPACE = 'datadog'
-
-const instrumented = Object.keys(require('../datadog-instrumentations/src/helpers/hooks.js'))
-const rawBuiltins = require('module').builtinModules
+const instrumentations = require('../datadog-instrumentations/src/helpers/instrumentations.js')
+const hooks = require('../datadog-instrumentations/src/helpers/hooks.js')
 
 warnIfUnsupported()
 
+for (const hook of Object.values(hooks)) {
+  hook()
+}
+
+const modulesOfInterest = new Set()
+
+for (const instrumentation of Object.values(instrumentations)) {
+  for (const entry of instrumentation) {
+    if (!entry.file) {
+      modulesOfInterest.add(entry.name) // e.g. "redis"
+    } else {
+      modulesOfInterest.add(`${entry.name}/${entry.file}`) // e.g. "redis/my/file.js"
+    }
+  }
+}
+
+const NAMESPACE = 'datadog'
+const NM = 'node_modules/'
+const INSTRUMENTED = Object.keys(instrumentations)
+const RAW_BUILTINS = require('module').builtinModules
+const CHANNEL = 'dd-trace:bundler:load'
+
 const builtins = new Set()
 
-for (const builtin of rawBuiltins) {
+for (const builtin of RAW_BUILTINS) {
   builtins.add(builtin)
   builtins.add(`node:${builtin}`)
 }
 
-const packagesOfInterest = new Set()
-
 const DEBUG = !!process.env.DD_TRACE_DEBUG
 
-// We don't want to handle any built-in packages via DCITM
+// We don't want to handle any built-in packages
 // Those packages will still be handled via RITM
 // Attempting to instrument them would fail as they have no package.json file
-for (const pkg of instrumented) {
+for (const pkg of INSTRUMENTED) {
   if (builtins.has(pkg)) continue
   if (pkg.startsWith('node:')) continue
-  packagesOfInterest.add(pkg)
+  modulesOfInterest.add(pkg)
 }
 
-const DC_CHANNEL = 'dd-trace:bundledModuleLoadStart'
-
 module.exports.name = 'datadog-esbuild'
 
 module.exports.setup = function (build) {
   build.onResolve({ filter: /.*/ }, args => {
+    let fullPathToModule
+    try {
+      fullPathToModule = dotFriendlyResolve(args.path, args.resolveDir)
+    } catch (err) {
+      console.warn(`Unable to find "${args.path}". Is the package dead code?`)
+      return
+    }
+    const extracted = extractPackageAndModulePath(fullPathToModule)
     const packageName = args.path
 
-    if (args.namespace === 'file' && packagesOfInterest.has(packageName)) {
+    const internal = builtins.has(args.path)
+
+    if (args.namespace === 'file' && (
+      modulesOfInterest.has(packageName) || modulesOfInterest.has(`${extracted.pkg}/${extracted.path}`))
+    ) {
       // The file namespace is used when requiring files from disk in userland
 
       let pathToPackageJson
       try {
-        pathToPackageJson = require.resolve(`${packageName}/package.json`, { paths: [ args.resolveDir ] })
+        pathToPackageJson = require.resolve(`${extracted.pkg}/package.json`, { paths: [ args.resolveDir ] })
       } catch (err) {
         if (err.code === 'MODULE_NOT_FOUND') {
-          console.warn(`Unable to open "${packageName}/package.json". Is the "${packageName}" package dead code?`)
+          if (!internal) {
+            console.warn(`Unable to find "${extracted.pkg}/package.json". Is the package dead code?`)
+          }
           return
         } else {
           throw err
         }
       }
 
-      const pkg = require(pathToPackageJson)
+      const packageJson = require(pathToPackageJson)
 
-      if (DEBUG) {
-        console.log(`resolve ${packageName}@${pkg.version}`)
-      }
+      if (DEBUG) console.log(`RESOLVE ${packageName}@${packageJson.version}`)
 
       // https://esbuild.github.io/plugins/#on-resolve-arguments
       return {
-        path: packageName,
+        path: fullPathToModule,
         namespace: NAMESPACE,
         pluginData: {
-          version: pkg.version
+          version: packageJson.version,
+          pkg: extracted.pkg,
+          path: extracted.path,
+          full: fullPathToModule,
+          raw: packageName,
+          internal
         }
       }
-    } else if (args.namespace === 'datadog') {
+    } else if (args.namespace === NAMESPACE) {
       // The datadog namespace is used when requiring files that are injected during the onLoad stage
-      // see note in onLoad
 
       if (builtins.has(packageName)) return
 
@@ -80,23 +112,28 @@ module.exports.setup = function (build) {
   })
 
   build.onLoad({ filter: /.*/, namespace: NAMESPACE }, args => {
-    if (DEBUG) {
-      console.log(`load ${args.path}@${args.pluginData.version}`)
-    }
+    const data = args.pluginData
+
+    if (DEBUG) console.log(`LOAD ${data.pkg}@${data.version}, pkg "${data.path}"`)
+
+    const path = data.raw !== data.pkg
+      ? `${data.pkg}/${data.path}`
+      : data.pkg
 
-    // JSON.stringify adds double quotes. For perf gain could simply add in quotes when we know it's safe.
     const contents = `
       const dc = require('diagnostics_channel');
-      const ch = dc.channel(${JSON.stringify(DC_CHANNEL + ':' + args.path)});
-      const mod = require(${JSON.stringify(args.path)});
+      const ch = dc.channel('${CHANNEL}');
+      const mod = require('${args.path}');
       const payload = {
         module: mod,
-        path: ${JSON.stringify(args.path)},
-        version: ${JSON.stringify(args.pluginData.version)}
+        version: '${data.version}',
+        package: '${data.pkg}',
+        path: '${path}'
       };
       ch.publish(payload);
       module.exports = payload.module;
     `
+
     // https://esbuild.github.io/plugins/#on-load-results
     return {
       contents,
@@ -121,3 +158,44 @@ function warnIfUnsupported () {
     console.error('more recent version is used at runtime, third party packages won\'t be instrumented.')
   }
 }
+
+// @see https://github.com/nodejs/node/issues/47000
+function dotFriendlyResolve (path, directory) {
+  if (path === '.') {
+    path = './'
+  } else if (path === '..') {
+    path = '../'
+  }
+
+  return require.resolve(path, { paths: [ directory ] })
+}
+
+/**
+ * For a given full path to a module,
+ *   return the package name it belongs to and the local path to the module
+ *   input: '/foo/node_modules/@co/stuff/foo/bar/baz.js'
+ *   output: { pkg: '@co/stuff', path: 'foo/bar/baz.js' }
+ */
+function extractPackageAndModulePath (fullPath) {
+  const nm = fullPath.lastIndexOf(NM)
+  if (nm < 0) {
+    return { pkg: null, path: null }
+  }
+
+  const subPath = fullPath.substring(nm + NM.length)
+  const firstSlash = subPath.indexOf('/')
+
+  if (subPath[0] === '@') {
+    const secondSlash = subPath.substring(firstSlash + 1).indexOf('/')
+
+    return {
+      pkg: subPath.substring(0, firstSlash + 1 + secondSlash),
+      path: subPath.substring(firstSlash + 1 + secondSlash + 1)
+    }
+  }
+
+  return {
+    pkg: subPath.substring(0, firstSlash),
+    path: subPath.substring(firstSlash + 1)
+  }
+}

package/packages/datadog-instrumentations/index.js

@@ -1,3 +1,4 @@
 'use strict'
 
+require('./src/helpers/bundler-register')
 require('./src/helpers/register')

package/packages/datadog-instrumentations/src/aws-sdk.js

@@ -177,7 +177,6 @@ addHook({ name: '@aws-sdk/smithy-client', versions: ['>=3'] }, smithy => {
 })
 
 addHook({ name: 'aws-sdk', versions: ['>=2.3.0'] }, AWS => {
-  shimmer.wrap(AWS.Request.prototype, 'promise', wrapRequest)
   shimmer.wrap(AWS.config, 'setPromisesDependency', setPromisesDependency => {
     return function wrappedSetPromisesDependency (dep) {
       const result = setPromisesDependency.apply(this, arguments)
@@ -188,9 +187,14 @@ addHook({ name: 'aws-sdk', versions: ['>=2.3.0'] }, AWS => {
   return AWS
 })
 
+addHook({ name: 'aws-sdk', file: 'lib/core.js', versions: ['>=2.3.0'] }, AWS => {
+  shimmer.wrap(AWS.Request.prototype, 'promise', wrapRequest)
+  return AWS
+})
+
 // <2.1.35 has breaking changes for instrumentation
 // https://github.com/aws/aws-sdk-js/pull/629
-addHook({ name: 'aws-sdk', versions: ['>=2.1.35'] }, AWS => {
+addHook({ name: 'aws-sdk', file: 'lib/core.js', versions: ['>=2.1.35'] }, AWS => {
   shimmer.wrap(AWS.Request.prototype, 'send', wrapRequest)
   return AWS
 })

package/packages/datadog-instrumentations/src/couchbase.js

@@ -160,7 +160,7 @@ function wrapV3Query (query) {
 }
 
 // semver >=2 <3
-addHook({ name: 'couchbase', file: 'lib/bucket.js', versions: ['^2.6.5'] }, Bucket => {
+addHook({ name: 'couchbase', file: 'lib/bucket.js', versions: ['^2.6.12'] }, Bucket => {
   const startCh = channel('apm:couchbase:query:start')
   const finishCh = channel('apm:couchbase:query:finish')
   const errorCh = channel('apm:couchbase:query:error')
@@ -208,7 +208,7 @@ addHook({ name: 'couchbase', file: 'lib/bucket.js', versions: ['^2.6.5'] }, Buck
   return Bucket
 })
 
-addHook({ name: 'couchbase', file: 'lib/cluster.js', versions: ['^2.6.5'] }, Cluster => {
+addHook({ name: 'couchbase', file: 'lib/cluster.js', versions: ['^2.6.12'] }, Cluster => {
   Cluster.prototype._maybeInvoke = wrapMaybeInvoke(Cluster.prototype._maybeInvoke)
   Cluster.prototype.query = wrapQuery(Cluster.prototype.query)
 
@@ -217,7 +217,7 @@ addHook({ name: 'couchbase', file: 'lib/cluster.js', versions: ['^2.6.5'] }, Clu
 
 // semver >=3 <3.2.0
 
-addHook({ name: 'couchbase', file: 'lib/collection.js', versions: ['>=3.0.0 <3.2.0'] }, Collection => {
+addHook({ name: 'couchbase', file: 'lib/collection.js', versions: ['^3.0.7', '^3.1.3'] }, Collection => {
   wrapAllNames(['upsert', 'insert', 'replace'], name => {
     shimmer.wrap(Collection.prototype, name, wrapWithName(name))
   })
@@ -225,7 +225,7 @@ addHook({ name: 'couchbase', file: 'lib/collection.js', versions: ['>=3.0.0 <3.2
   return Collection
 })
 
-addHook({ name: 'couchbase', file: 'lib/cluster.js', versions: ['>=3.0.0 <3.2.0'] }, Cluster => {
+addHook({ name: 'couchbase', file: 'lib/cluster.js', versions: ['^3.0.7', '^3.1.3'] }, Cluster => {
   shimmer.wrap(Cluster.prototype, 'query', wrapV3Query)
   return Cluster
 })

package/packages/datadog-instrumentations/src/fetch.js

@@ -17,29 +17,32 @@ function wrapFetch (fetch, Request) {
     const headers = req.headers
     const message = { req, headers }
 
-    startChannel.publish(message)
-
-    // Request object is read-only so we need new objects to change headers.
-    arguments[0] = message.req
-    arguments[1] = { headers: message.headers }
-
-    return fetch.apply(this, arguments)
-      .then(
-        res => {
-          finishChannel.publish({ req, res })
-
-          return res
-        },
-        err => {
-          if (err.name !== 'AbortError') {
-            errorChannel.publish(err)
-          }
+    return startChannel.runStores(message, () => {
+      // Request object is read-only so we need new objects to change headers.
+      arguments[0] = message.req
+      arguments[1] = { headers: message.headers }
+
+      return fetch.apply(this, arguments)
+        .then(
+          res => {
+            message.res = res
+
+            finishChannel.publish(message)
 
-          finishChannel.publish({ req })
+            return res
+          },
+          err => {
+            if (err.name !== 'AbortError') {
+              message.error = err
+              errorChannel.publish(message)
+            }
 
-          throw err
-        }
-      )
+            finishChannel.publish(message)
+
+            throw err
+          }
+        )
+    })
   }
 }
 

package/packages/datadog-instrumentations/src/helpers/bundler-register.js

@@ -0,0 +1,54 @@
+'use strict'
+
+// eslint-disable-next-line n/no-restricted-require
+const dc = require('diagnostics_channel')
+
+const {
+  filename,
+  loadChannel,
+  matchVersion
+} = require('./register.js')
+const hooks = require('./hooks')
+const instrumentations = require('./instrumentations')
+const log = require('../../../dd-trace/src/log')
+
+const CHANNEL = 'dd-trace:bundler:load'
+
+if (!dc.subscribe) {
+  dc.subscribe = (channel, cb) => {
+    dc.channel(channel).subscribe(cb)
+  }
+}
+if (!dc.unsubscribe) {
+  dc.unsubscribe = (channel, cb) => {
+    if (dc.channel(channel).hasSubscribers) {
+      dc.channel(channel).unsubscribe(cb)
+    }
+  }
+}
+
+dc.subscribe(CHANNEL, (payload) => {
+  try {
+    hooks[payload.package]()
+  } catch (err) {
+    log.error(`esbuild-wrapped ${payload.package} missing in list of hooks`)
+    throw err
+  }
+
+  if (!instrumentations[payload.package]) {
+    log.error(`esbuild-wrapped ${payload.package} missing in list of instrumentations`)
+    return
+  }
+
+  for (const { name, file, versions, hook } of instrumentations[payload.package]) {
+    if (payload.path !== filename(name, file)) continue
+    if (!matchVersion(payload.version, versions)) continue
+
+    try {
+      loadChannel.publish({ name, version: payload.version, file })
+      payload.module = hook(payload.module, payload.version)
+    } catch (e) {
+      log.error(e)
+    }
+  }
+})

package/packages/datadog-instrumentations/src/helpers/hook.js

@@ -3,10 +3,9 @@
 const path = require('path')
 const iitm = require('../../../dd-trace/src/iitm')
 const ritm = require('../../../dd-trace/src/ritm')
-const dcitm = require('../../../dd-trace/src/dcitm')
 
 /**
- * This is called for every module that dd-trace supports instrumentation for.
+ * This is called for every package/internal-module that dd-trace supports instrumentation for
  * In practice, `modules` is always an array with a single entry.
  *
  * @param {string[]} modules list of modules to hook into
@@ -41,13 +40,11 @@ function Hook (modules, onrequire) {
       return safeHook(moduleExports, moduleName, moduleBaseDir)
     }
   })
-  this._dcitmHook = dcitm(modules, {}, safeHook)
 }
 
 Hook.prototype.unhook = function () {
   this._ritmHook.unhook()
   this._iitmHook.unhook()
-  this._dcitmHook.unhook()
   this._patched = Object.create(null)
 }
 

package/packages/datadog-instrumentations/src/helpers/hooks.js

@@ -11,6 +11,8 @@ module.exports = {
   '@hapi/hapi': () => require('../hapi'),
   '@jest/core': () => require('../jest'),
   '@jest/reporters': () => require('../jest'),
+  '@jest/test-sequencer': () => require('../jest'),
+  '@jest/transform': () => require('../jest'),
   '@koa/router': () => require('../koa'),
   '@node-redis/client': () => require('../redis'),
   '@opensearch-project/opensearch': () => require('../opensearch'),
@@ -38,6 +40,7 @@ module.exports = {
   'find-my-way': () => require('../find-my-way'),
   'fs': () => require('../fs'),
   'node:fs': () => require('../fs'),
+  'generic-pool': () => require('../generic-pool'),
   'graphql': () => require('../graphql'),
   'grpc': () => require('../grpc'),
   'hapi': () => require('../hapi'),
@@ -51,6 +54,7 @@ module.exports = {
   'jest-environment-jsdom': () => require('../jest'),
   'jest-jasmine2': () => require('../jest'),
   'jest-worker': () => require('../jest'),
+  'knex': () => require('../knex'),
   'koa': () => require('../koa'),
   'koa-router': () => require('../koa'),
   'kafkajs': () => require('../kafkajs'),

package/packages/datadog-instrumentations/src/helpers/register.js

@@ -20,7 +20,9 @@ const disabledInstrumentations = new Set(
 const loadChannel = channel('dd-trace:instrumentation:load')
 
 // Globals
-require('../fetch')
+if (!disabledInstrumentations.has('fetch')) {
+  require('../fetch')
+}
 
 // TODO: make this more efficient
 
@@ -30,6 +32,7 @@ for (const packageName of names) {
   Hook([packageName], (moduleExports, moduleName, moduleBaseDir, moduleVersion) => {
     moduleName = moduleName.replace(pathSepExpr, '/')
 
+    // This executes the integration file thus adding its entries to `instrumentations`
     hooks[packageName]()
 
     if (!instrumentations[packageName]) {
@@ -74,5 +77,7 @@ function filename (name, file) {
 
 module.exports = {
   filename,
-  pathSepExpr
+  pathSepExpr,
+  loadChannel,
+  matchVersion
 }