import-in-the-middle 1.7.1 → 1.7.3

package/.eslintrc.yaml

@@ -0,0 +1,23 @@
+overrides:
+  - files:
+    - '**/*.{js,cjs,mjs}'
+
+parser: '@babel/eslint-parser'
+parserOptions:
+  ecmaVersion: latest
+  requireConfigFile: false
+  sourceType: 'script'
+  babelOptions:
+    plugins:
+      - '@babel/plugin-syntax-import-assertions'
+
+rules:
+  "import/first": off
+
+extends:
+  - "standard"
+
+ignorePatterns:
+  - test/fixtures/circular-a.js
+  - test/fixtures/circular-b.js
+  - test/fixtures/reexport.js

package/hook.js

@@ -2,8 +2,9 @@
 //
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc.
 
+const { randomBytes } = require('crypto')
 const specifiers = new Map()
-const isWin = process.platform === "win32"
+const isWin = process.platform === 'win32'
 
 // FIXME: Typescript extensions are added temporarily until we find a better
 // way of supporting arbitrary extensions
@@ -15,10 +16,10 @@ const NODE_MINOR = Number(NODE_VERSION[1])
 let entrypoint
 
 let getExports
-if (NODE_MAJOR >= 20 || (NODE_MAJOR == 18 && NODE_MINOR >= 19)) {
+if (NODE_MAJOR >= 20 || (NODE_MAJOR === 18 && NODE_MINOR >= 19)) {
   getExports = require('./lib/get-exports.js')
 } else {
-  getExports = (url) => import(url).then(Object.keys)
+  getExports = ({ url }) => import(url).then(Object.keys)
 }
 
 function hasIitm (url) {
@@ -55,7 +56,7 @@ function deleteIitm (url) {
   return resultUrl
 }
 
-function isNode16AndBiggerOrEqualsThan16_17_0() {
+function isNodeMajor16AndMinor17OrGreater () {
   return NODE_MAJOR === 16 && NODE_MINOR >= 17
 }
 
@@ -67,16 +68,159 @@ function isNodeProtocol (urlObj) {
   return urlObj.protocol === 'node:'
 }
 
-function needsToAddFileProtocol(urlObj) {
+function needsToAddFileProtocol (urlObj) {
   if (NODE_MAJOR === 17) {
     return !isFileProtocol(urlObj)
   }
-  if (isNode16AndBiggerOrEqualsThan16_17_0()) {
+  if (isNodeMajor16AndMinor17OrGreater()) {
     return !isFileProtocol(urlObj) && !isNodeProtocol(urlObj)
   }
   return !isFileProtocol(urlObj) && NODE_MAJOR < 18
 }
 
+/**
+ * Determines if a specifier represents an export all ESM line.
+ * Note that the expected `line` isn't 100% valid ESM. It is derived
+ * from the `getExports` function wherein we have recognized the true
+ * line and re-mapped it to one we expect.
+ *
+ * @param {string} line
+ * @returns {boolean}
+ */
+function isStarExportLine (line) {
+  return /^\* from /.test(line)
+}
+
+/**
+ * @typedef {object} ProcessedModule
+ * @property {string[]} imports A set of ESM import lines to be added to the
+ * shimmed module source.
+ * @property {string[]} namespaces A set of identifiers representing the
+ * modules in `imports`, e.g. for `import * as foo from 'bar'`, "foo" will be
+ * present in this array.
+ * @property {Map<string, string>} setters The shimmed setters for all the
+ * exports from the module and any transitive export all modules. The key is
+ * used to deduplicate conflicting exports, assigning a priority to `default`
+ * exports.
+ */
+
+/**
+ * Processes a module's exports and builds a set of new import statements,
+ * namespace names, and setter blocks. If an export all export if encountered,
+ * the target exports will be hoisted to the current module via a generated
+ * namespace.
+ *
+ * @param {object} params
+ * @param {string} params.srcUrl The full URL to the module to process.
+ * @param {object} params.context Provided by the loaders API.
+ * @param {Function} params.parentGetSource Provides the source code for the
+ * parent module.
+ * @param {string} [params.ns='namespace'] A string identifier that will be
+ * used as the namespace for the identifiers exported by the module.
+ * @param {string} [params.defaultAs='default'] The name to give the default
+ * identifier exported by the module (if one exists). This is really only
+ * useful in a recursive situation where a transitive module's default export
+ * needs to be renamed to the name of the module.
+ *
+ * @returns {Promise<ProcessedModule>}
+ */
+async function processModule ({
+  srcUrl,
+  context,
+  parentGetSource,
+  ns = 'namespace',
+  defaultAs = 'default'
+}) {
+  const exportNames = await getExports({
+    url: srcUrl,
+    context,
+    parentLoad: parentGetSource,
+    defaultAs
+  })
+  const imports = [`import * as ${ns} from ${JSON.stringify(srcUrl)}`]
+  const namespaces = [ns]
+
+  // As we iterate found module exports we will add setter code blocks
+  // to this map that will eventually be inserted into the shim module's
+  // source code. We utilize a map in order to prevent duplicate exports.
+  // As a consequence of default renaming, it is possible that a file named
+  // `foo.mjs` which has `export function foo() {}` and `export default foo`
+  // exports will result in the "foo" export being defined twice in our shim.
+  // The map allows us to avoid this situation at the cost of losing the
+  // named export in favor of the default export.
+  const setters = new Map()
+
+  for (const n of exportNames) {
+    if (isStarExportLine(n) === true) {
+      const [, modFile] = n.split('* from ')
+      const normalizedModName = normalizeModName(modFile)
+      const modUrl = new URL(modFile, srcUrl).toString()
+      const modName = Buffer.from(modFile, 'hex') + Date.now() + randomBytes(4).toString('hex')
+
+      const data = await processModule({
+        srcUrl: modUrl,
+        context,
+        parentGetSource,
+        ns: `$${modName}`,
+        defaultAs: normalizedModName
+      })
+      Array.prototype.push.apply(imports, data.imports)
+      Array.prototype.push.apply(namespaces, data.namespaces)
+      for (const [k, v] of data.setters.entries()) {
+        setters.set(k, v)
+      }
+
+      continue
+    }
+
+    const matches = /^rename (.+) as (.+)$/.exec(n)
+    if (matches !== null) {
+      // Transitive modules that export a default identifier need to have
+      // that identifier renamed to the name of module. And our shim setter
+      // needs to utilize that new name while being initialized from the
+      // corresponding origin namespace.
+      const renamedExport = matches[2]
+      setters.set(`$${renamedExport}${ns}`, `
+      let $${renamedExport} = ${ns}.default
+      export { $${renamedExport} as ${renamedExport} }
+      set.${renamedExport} = (v) => {
+        $${renamedExport} = v
+        return true
+      }
+      `)
+      continue
+    }
+
+    setters.set(`$${n}` + ns, `
+    let $${n} = ${ns}.${n}
+    export { $${n} as ${n} }
+    set.${n} = (v) => {
+      $${n} = v
+      return true
+    }
+    `)
+  }
+
+  return { imports, namespaces, setters }
+}
+
+/**
+ * Given a module name, e.g. 'foo-bar' or './foo-bar.js', normalize it to a
+ * string that is a valid JavaScript identifier, e.g. `fooBar`. Normalization
+ * means converting kebab-case to camelCase while removing any path tokens and
+ * file extensions.
+ *
+ * @param {string} name The module name to normalize.
+ *
+ * @returns {string} The normalized identifier.
+ */
+function normalizeModName (name) {
+  return name
+    .split('/')
+    .pop()
+    .replace(/(.+)\.(?:js|mjs)$/, '$1')
+    .replaceAll(/(-.)/g, x => x[1].toUpperCase())
+}
 
 function addIitm (url) {
   const urlObj = new URL(url)
@@ -109,7 +253,6 @@ function createHook (meta) {
       return url
     }
 
-
     specifiers.set(url.url, specifier)
 
     return {
@@ -123,21 +266,61 @@ function createHook (meta) {
   async function getSource (url, context, parentGetSource) {
     if (hasIitm(url)) {
       const realUrl = deleteIitm(url)
-      const exportNames = await getExports(realUrl, context, parentGetSource)
+      const { imports, namespaces, setters: mapSetters } = await processModule({
+        srcUrl: realUrl,
+        context,
+        parentGetSource
+      })
+      const setters = Array.from(mapSetters.values())
+
+      // When we encounter modules that re-export all identifiers from other
+      // modules, it is possible that the transitive modules export a default
+      // identifier. Due to us having to merge all transitive modules into a
+      // single common namespace, we need to recognize these default exports
+      // and remap them to a name based on the module name. This prevents us
+      // from overriding the top-level module's (the one actually being imported
+      // by some source code) default export when we merge the namespaces.
+      const renamedDefaults = setters
+        .map(s => {
+          const matches = /let \$(.+) = (\$.+)\.default/.exec(s)
+          if (matches === null) return undefined
+          return `_['${matches[1]}'] = ${matches[2]}.default`
+        })
+        .filter(s => s)
+
+      // The for loops are how we merge namespaces into a common namespace that
+      // can be proxied. We can't use a simple `Object.assign` style merging
+      // because transitive modules can export a default identifier that would
+      // override the desired default identifier. So we need to do manual
+      // merging with some logic around default identifiers.
+      //
+      // Additionally, we need to make sure any renamed default exports in
+      // transitive dependencies are added to the common namespace. This is
+      // accomplished through the `renamedDefaults` array.
       return {
         source: `
 import { register } from '${iitmURL}'
-import * as namespace from ${JSON.stringify(url)}
+${imports.join('\n')}
+
+const namespaces = [${namespaces.join(', ')}]
+const _ = {}
 const set = {}
-${exportNames.map((n) => `
-let $${n} = namespace.${n}
-export { $${n} as ${n} }
-set.${n} = (v) => {
-  $${n} = v
-  return true
+
+const primary = namespaces.shift()
+for (const [k, v] of Object.entries(primary)) {
+  _[k] = v
 }
-`).join('\n')}
-register(${JSON.stringify(realUrl)}, namespace, set, ${JSON.stringify(specifiers.get(realUrl))})
+for (const ns of namespaces) {
+  for (const [k, v] of Object.entries(ns)) {
+    if (k === 'default') continue
+    _[k] = v
+  }
+}
+
+${setters.join('\n')}
+${renamedDefaults.join('\n')}
+
+register(${JSON.stringify(realUrl)}, _, set, ${JSON.stringify(specifiers.get(realUrl))})
 `
       }
     }

package/index.js

@@ -12,26 +12,26 @@ const {
   toHook
 } = require('./lib/register')
 
-function addHook(hook) {
+function addHook (hook) {
   importHooks.push(hook)
   toHook.forEach(([name, namespace]) => hook(name, namespace))
 }
 
-function removeHook(hook) {
+function removeHook (hook) {
   const index = importHooks.indexOf(hook)
   if (index > -1) {
     importHooks.splice(index, 1)
   }
 }
 
-function callHookFn(hookFn, namespace, name, baseDir) {
+function callHookFn (hookFn, namespace, name, baseDir) {
   const newDefault = hookFn(namespace, name, baseDir)
   if (newDefault && newDefault !== namespace) {
     namespace.default = newDefault
   }
 }
 
-function Hook(modules, options, hookFn) {
+function Hook (modules, options, hookFn) {
   if ((this instanceof Hook) === false) return new Hook(modules, options, hookFn)
   if (typeof modules === 'function') {
     hookFn = modules

package/lib/get-esm-exports.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { Parser } = require('acorn')
-const { importAssertions } = require('acorn-import-assertions');
+const { importAssertions } = require('acorn-import-assertions')
 
 const acornOpts = {
   ecmaVersion: 'latest',
@@ -14,9 +14,36 @@ function warn (txt) {
   process.emitWarning(txt, 'get-esm-exports')
 }
 
-function getEsmExports (moduleStr) {
+/**
+ * Utilizes an AST parser to interpret ESM source code and build a list of
+ * exported identifiers. In the baseline case, the list of identifiers will be
+ * the simple identifier names as written in the source code of the module.
+ * However, there are some special cases:
+ *
+ * 1. When an `export * from './foo.js'` line is encountered it is rewritten
+ * as `* from ./foo.js`. This allows the interpreting code to recognize a
+ * transitive export and recursively parse the indicated module. The returned
+ * identifier list will have "* from ./foo.js" as an item.
+ *
+ * 2. When `defaultAs` has a value other than 'default', the export line will
+ * be rewritten as `rename <identifier> as <defaultAsValue>`. This rename string
+ * will be an item in the returned identifier list.
+ *
+ * @param {object} params
+ * @param {string} params.moduleSource The source code of the module to parse
+ * and interpret.
+ * @param {string} [defaultAs='default'] When anything other than 'default' any
+ * `export default` lines will be rewritten utilizing the value provided. For
+ * example, if a module 'foo-bar.js' has the line `export default foo` and the
+ * value of this parameter is 'baz', then the export will be rewritten to
+ * `rename foo as baz`.
+ *
+ * @returns {string[]} The identifiers exported by the module along with any
+ * custom directives.
+ */
+function getEsmExports ({ moduleSource, defaultAs = 'default' }) {
   const exportedNames = new Set()
-  const tree = parser.parse(moduleStr, acornOpts)
+  const tree = parser.parse(moduleSource, acornOpts)
   for (const node of tree.body) {
     if (!node.type.startsWith('Export')) continue
     switch (node.type) {
@@ -27,14 +54,29 @@ function getEsmExports (moduleStr) {
           parseSpecifiers(node, exportedNames)
         }
         break
-      case 'ExportDefaultDeclaration':
-        exportedNames.add('default')
+
+      case 'ExportDefaultDeclaration': {
+        if (defaultAs === 'default') {
+          exportedNames.add('default')
+          break
+        }
+
+        if (node.declaration.type.toLowerCase() === 'identifier') {
+          // e.g. `export default foo`
+          exportedNames.add(`rename ${node.declaration.name} as ${defaultAs}`)
+        } else {
+          // e.g. `export function foo () {}
+          exportedNames.add(`rename ${node.declaration.id.name} as ${defaultAs}`)
+        }
+
         break
+      }
+
       case 'ExportAllDeclaration':
         if (node.exported) {
           exportedNames.add(node.exported.name)
         } else {
-          exportedNames.add('*')
+          exportedNames.add(`* from ${node.source.value}`)
         }
         break
       default:

package/lib/get-exports.js

@@ -3,13 +3,60 @@
 const getEsmExports = require('./get-esm-exports.js')
 const { parse: getCjsExports } = require('cjs-module-lexer')
 const fs = require('fs')
-const { fileURLToPath } = require('url')
+const { fileURLToPath, pathToFileURL } = require('url')
 
-function addDefault(arr) {
+function addDefault (arr) {
   return Array.from(new Set(['default', ...arr]))
 }
 
-async function getExports (url, context, parentLoad) {
+const urlsBeingProcessed = new Set() // Guard against circular imports.
+
+async function getFullCjsExports (url, context, parentLoad, source) {
+  if (urlsBeingProcessed.has(url)) {
+    return []
+  }
+  urlsBeingProcessed.add(url)
+
+  const ex = getCjsExports(source)
+  const full = Array.from(new Set([
+    ...addDefault(ex.exports),
+    ...(await Promise.all(ex.reexports.map(re => getExports(({
+      url: (/^(..?($|\/|\\))/).test(re)
+        ? pathToFileURL(require.resolve(fileURLToPath(new URL(re, url)))).toString()
+        : pathToFileURL(require.resolve(re)).toString(),
+      context,
+      parentLoad
+    }))))).flat()
+  ]))
+
+  urlsBeingProcessed.delete(url)
+  return full
+}
+
+/**
+ * Inspects a module for its type (commonjs or module), attempts to get the
+ * source code for said module from the loader API, and parses the result
+ * for the entities exported from that module.
+ *
+ * @param {object} params
+ * @param {string} params.url A file URL string pointing to the module that
+ * we should get the exports of.
+ * @param {object} params.context Context object as provided by the `load`
+ * hook from the loaders API.
+ * @param {Function} params.parentLoad Next hook function in the loaders API
+ * hook chain.
+ * @param {string} [defaultAs='default'] When anything other than 'default',
+ * will trigger remapping of default exports in ESM source files to the
+ * provided name. For example, if a submodule has `export default foo` and
+ * 'myFoo' is provided for this parameter, the export line will be rewritten
+ * to `rename foo as myFoo`. This is key to being able to support
+ * `export * from 'something'` exports.
+ *
+ * @returns {Promise<string[]>} An array of identifiers exported by the module.
+ * Please see {@link getEsmExports} for caveats on special identifiers that may
+ * be included in the result set.
+ */
+async function getExports ({ url, context, parentLoad, defaultAs = 'default' }) {
   // `parentLoad` gives us the possibility of getting the source
   // from an upstream loader. This doesn't always work though,
   // so later on we fall back to reading it from disk.
@@ -30,21 +77,21 @@ async function getExports (url, context, parentLoad) {
   }
 
   if (format === 'module') {
-    return getEsmExports(source)
+    return getEsmExports({ moduleSource: source, defaultAs })
   }
   if (format === 'commonjs') {
-    return addDefault(getCjsExports(source).exports)
+    return getFullCjsExports(url, context, parentLoad, source)
   }
 
   // At this point our `format` is either undefined or not known by us. Fall
   // back to parsing as ESM/CJS.
-  const esmExports = getEsmExports(source)
+  const esmExports = getEsmExports({ moduleSource: source, defaultAs })
   if (!esmExports.length) {
     // TODO(bengl) it's might be possible to get here if somehow the format
     // isn't set at first and yet we have an ESM module with no exports.
     // I couldn't construct an example that would do this, so maybe it's
     // impossible?
-    return addDefault(getCjsExports(source).exports)
+    return getFullCjsExports(url, context, parentLoad, source)
   }
 }
 

package/lib/register.js

@@ -2,18 +2,17 @@
 //
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc.
 
-
 const importHooks = [] // TODO should this be a Set?
 const setters = new WeakMap()
 const specifiers = new Map()
 const toHook = []
 
 const proxyHandler = {
-  set(target, name, value) {
+  set (target, name, value) {
     return setters.get(target)[name](value)
   },
 
-  defineProperty(target, property, descriptor) {
+  defineProperty (target, property, descriptor) {
     if ((!('value' in descriptor))) {
       throw new Error('Getters/setters are not supported for exports property descriptors.')
     }
@@ -22,7 +21,7 @@ const proxyHandler = {
   }
 }
 
-function register(name, namespace, set, specifier) {
+function register (name, namespace, set, specifier) {
   specifiers.set(name, specifier)
   setters.set(namespace, set)
   const proxy = new Proxy(namespace, proxyHandler)

package/package.json

@@ -1,12 +1,14 @@
 {
   "name": "import-in-the-middle",
-  "version": "1.7.1",
+  "version": "1.7.3",
   "description": "Intercept imports in Node.js",
   "main": "index.js",
   "scripts": {
-    "test": "c8 --check-coverage --lines 85 imhotap --runner 'node test/runtest' --files test/{hook,low-level,other,get-esm-exports}/*",
-    "test:ts": "c8 imhotap --runner 'node test/runtest' --files test/typescript/*.test.mts",
-    "coverage": "c8 --reporter html imhotap --runner 'node test/runtest' --files test/{hook,low-level,other,get-esm-exports}/* && echo '\nNow open coverage/index.html\n'"
+    "test": "c8 --reporter lcov --check-coverage --lines 50 imhotap --files test/{hook,low-level,other,get-esm-exports}/*",
+    "test:ts": "c8 --reporter lcov imhotap --files test/typescript/*.test.mts",
+    "coverage": "c8 --reporter html imhotap --files test/{hook,low-level,other,get-esm-exports}/* && echo '\nNow open coverage/index.html\n'",
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix"
   },
   "repository": {
     "type": "git",
@@ -26,9 +28,22 @@
     "url": "https://github.com/DataDog/import-in-the-middle/issues"
   },
   "homepage": "https://github.com/DataDog/import-in-the-middle#readme",
+  "imhotap": {
+    "runner": "node",
+    "test-env": "NODE_OPTIONS=--no-warnings --require ./test/version-check.js --experimental-loader ./test/generic-loader.mjs"
+  },
   "devDependencies": {
+    "@babel/core": "^7.23.7",
+    "@babel/eslint-parser": "^7.23.3",
+    "@babel/plugin-syntax-import-assertions": "^7.23.3",
     "@types/node": "^18.0.6",
     "c8": "^7.8.0",
+    "eslint": "^8.55.0",
+    "eslint-config-standard": "^17.1.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-n": "^16.4.0",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-promise": "^6.1.1",
     "imhotap": "^2.1.0",
     "ts-node": "^10.9.1",
     "typescript": "^4.7.4"

package/test/README.md

@@ -3,11 +3,15 @@ These tests are organized as follows:
 * Located in the `hook` directory if they use the `Hook` class.
 * Located in the `low-level` directory if they use the "low-level" API,
   `addHook` and `removeHook`.
+* Other tests are in other adjacent directories.
 
-The tests should be run with the `runtest` command found in this directory. If
-the command exits with a non-zero code, then it's a test failure.
+The tests can be run individually as Node.js programs with non-zero exit codes
+upon failures. They should be run with the following Node.js command-line
+options (assuming they're run from the project root):
 
-Running of all the tests can be done with `npm test`.
+```
+--require ./test/version-check.js
+--experimental-loader ./test/generic-loader.mmjs
+```
 
-Coverage must be 100% according to `c8`. If you don't have 100% coverage, you
-can run `npm run coverage` to get coverage data in HTML form.
+The entire test suite can be run with `npm test`.

package/test/fixtures/a.mjs

@@ -0,0 +1,7 @@
+export const a = 'a'
+
+export function aFunc () {
+  return a
+}
+
+export * from './foo.mjs'

package/test/fixtures/b.mjs

@@ -0,0 +1,5 @@
+export const b = 'b'
+
+export function bFunc () {
+  return b
+}

package/test/fixtures/bundle.mjs

@@ -0,0 +1,4 @@
+import bar from './something.mjs'
+export default bar
+export * from './a.mjs'
+export * from './b.mjs'

package/test/fixtures/circular-a.js

@@ -0,0 +1,18 @@
+// The following is generated by tslib. __exportStar is a format which cjs-module-lexer exposes as a reexport
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+  if (k2 === undefined) k2 = k;
+  Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+  if (k2 === undefined) k2 = k;
+  o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+
+
+__exportStar(require("./circular-b"), exports);
+
+exports.foo = 42;
+

package/test/fixtures/circular-b.js

@@ -0,0 +1,15 @@
+// The following is generated by tslib. __exportStar is a format which cjs-module-lexer exposes as a reexport
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+  if (k2 === undefined) k2 = k;
+  Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+  if (k2 === undefined) k2 = k;
+  o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+
+__exportStar(require("./circular-a"), exports);
+

package/test/fixtures/cyclical-a.mjs

@@ -0,0 +1,7 @@
+import { testB } from './cyclical-b.mjs'
+
+export function testA () {
+  console.log('testA')
+}
+
+testB()

package/test/fixtures/cyclical-b.mjs

@@ -0,0 +1,6 @@
+import { testA } from './cyclical-a.mjs'
+
+export function testB () {
+  console.log('testB')
+  testA()
+}

package/test/fixtures/default-class.mjs

@@ -0,0 +1,3 @@
+export default class DefaultClass {
+  value = 'DefaultClass'
+}