bare-module-traverse 2.2.0 → 2.3.1

package/README.md

@@ -67,10 +67,13 @@ options = {
     // rewritten to match, so `'.ts': '.js'` yields a `file:///foo.js`
     // dependency rather than `file:///foo.ts`.
   },
-  resolve: resolve.default
+  resolve: resolve.default,
+  visited: new Set()
 }
 ```
 
+`visited` is a `Set` of already visited module hrefs. If provided, modules whose href is already in the set are skipped and the set is updated in place as traversal proceeds. This allows a single set to be shared across several traversals so that a later traversal, such as one rooted at a dynamically imported module, does not revisit modules already seen by an earlier one.
+
 Options supported by <https://github.com/holepunchto/bare-module-resolve> and <https://github.com/holepunchto/bare-addon-resolve> may also be specified.
 
 #### `for (const dependency of dependencies)`
@@ -81,12 +84,16 @@ Synchronously iterate the module graph. Each yielded dependency has the followin
 dependency = {
   url: URL,
   source: 'string' | Buffer, // Source as returned by `readModule()`
+  type: constants.SCRIPT, // The detected module type, or `0` if unknown
   imports: {
     // See https://github.com/holepunchto/bare-module#imports
   },
   lexer: {
     imports: [
       // See https://github.com/holepunchto/bare-module-lexer#api
+    ],
+    exports: [
+      // See https://github.com/holepunchto/bare-module-lexer#api
     ]
   }
 }
@@ -96,6 +103,20 @@ dependency = {
 
 Asynchronously iterate the module graph. If `readModule` returns a promise or `listPrefix` returns a promise generator, these will be awaited. The same comments as `for (const dependency of dependencies)` apply.
 
+#### `constants`
+
+The module type constants used by the `type` field of each dependency and accepted as the `defaultType` option.
+
+| Constant | Description        |
+| :------- | :----------------- |
+| `SCRIPT` | A CommonJS module. |
+| `MODULE` | An ES module.      |
+| `JSON`   | A JSON module.     |
+| `BUNDLE` | A bundle module.   |
+| `ADDON`  | A native addon.    |
+| `BINARY` | A binary module.   |
+| `TEXT`   | A text module.     |
+
 ### Resolution
 
 Module and addon resolution is configurable by providing a resolver function. A resolver function is a generator function that yields values matching the shapes defined by <https://github.com/holepunchto/bare-module-resolve#algorithm>. Several resolvers are provided out of the box to support the most common use cases.
@@ -162,7 +183,8 @@ next.value = {
 
 ```js
 next.value = {
-  children: Generator
+  children: Generator,
+  deferred: boolean
 }
 ```
 
@@ -173,12 +195,16 @@ next.value = {
   dependency: {
     url: URL,
     source: 'string' | Buffer,
+    type: constants.SCRIPT,
     imports: {
       // See https://github.com/holepunchto/bare-module#imports
     },
     lexer: {
       imports: [
         // See https://github.com/holepunchto/bare-module-lexer#api
+      ],
+      exports: [
+        // See https://github.com/holepunchto/bare-module-lexer#api
       ]
     }
   }

package/index.d.ts

@@ -6,14 +6,16 @@ import {
   type ResolutionsMap
 } from 'bare-module-resolve'
 import { type ResolveOptions, type Resolver } from 'bare-addon-resolve'
-import { type Import } from 'bare-module-lexer'
+import { type Import, type Export } from 'bare-module-lexer'
 
 interface Dependency {
   url: URL
   source: string | Buffer
+  type: number
   imports: ImportsMap
   lexer: {
     imports: Import[]
+    exports: Export[]
   }
 }
 
@@ -67,7 +69,7 @@ declare namespace traverse {
   export { type TraverseOptions }
 
   export type Traversal = Generator<
-    { module: URL } | { prefix: URL } | { children: URL } | { dependency: URL },
+    { module: URL } | { prefix: URL } | { children: URL; deferred: boolean } | { dependency: URL },
     boolean,
     void | URL[] | Buffer | string | null
   >
@@ -85,7 +87,6 @@ declare namespace traverse {
     ADDON: number
     BINARY: number
     TEXT: number
-    ASSET: number
   }
 
   export function module(

package/index.js

@@ -10,8 +10,7 @@ const constants = {
   BUNDLE: 4,
   ADDON: 5,
   BINARY: 6,
-  TEXT: 7,
-  ASSET: 8
+  TEXT: 7
 }
 
 module.exports = exports = function traverse(entry, opts, readModule, listPrefix) {
@@ -25,10 +24,17 @@ module.exports = exports = function traverse(entry, opts, readModule, listPrefix
     *[Symbol.iterator]() {
       const artifacts = { addons: [], assets: [] }
 
-      const queue = [exports.module(entry, null, {}, artifacts, new Set(), opts)]
+      const visited = opts.visited || new Set()
 
-      while (queue.length > 0) {
-        const generator = queue.pop()
+      const queue = [exports.module(entry, null, {}, artifacts, visited, opts)]
+
+      // Asset traversal is deferred until the module graph has been fully
+      // drained so that a module imported both as an asset and as a regular
+      // module is typed by the regular import
+      const deferred = []
+
+      while (queue.length > 0 || deferred.length > 0) {
+        const generator = queue.length > 0 ? queue.pop() : deferred.shift()
 
         let next = generator.next()
 
@@ -52,8 +58,10 @@ module.exports = exports = function traverse(entry, opts, readModule, listPrefix
 
             next = generator.next(result)
           } else {
-            if (value.children) queue.push(value.children)
-            else yield value.dependency
+            if (value.children) {
+              if (value.deferred) deferred.push(value.children)
+              else queue.push(value.children)
+            } else yield value.dependency
 
             next = generator.next()
           }
@@ -66,10 +74,17 @@ module.exports = exports = function traverse(entry, opts, readModule, listPrefix
     async *[Symbol.asyncIterator]() {
       const artifacts = { addons: [], assets: [] }
 
-      const queue = [exports.module(entry, null, {}, artifacts, new Set(), opts)]
+      const visited = opts.visited || new Set()
+
+      const queue = [exports.module(entry, null, {}, artifacts, visited, opts)]
 
-      while (queue.length > 0) {
-        const generator = queue.pop()
+      // Asset traversal is deferred until the module graph has been fully
+      // drained so that a module imported both as an asset and as a regular
+      // module is typed by the regular import.
+      const deferred = []
+
+      while (queue.length > 0 || deferred.length > 0) {
+        const generator = queue.length > 0 ? queue.pop() : deferred.shift()
 
         let next = generator.next()
 
@@ -93,8 +108,10 @@ module.exports = exports = function traverse(entry, opts, readModule, listPrefix
 
             next = generator.next(result)
           } else {
-            if (value.children) queue.push(value.children)
-            else yield value.dependency
+            if (value.children) {
+              if (value.deferred) deferred.push(value.children)
+              else queue.push(value.children)
+            } else yield value.dependency
 
             next = generator.next()
           }
@@ -130,10 +147,12 @@ exports.alias = function alias(url, opts = {}) {
 }
 
 exports.module = function* (url, source, attributes, artifacts, visited, opts = {}) {
-  const { resolutions = null, defaultType = constants.SCRIPT, aliases = null } = opts
+  const { resolutions = null, asset = false } = opts
 
   if (visited.has(url.href)) return false
 
+  visited.add(url.href)
+
   if (source === null) {
     source = yield { module: url }
 
@@ -142,8 +161,6 @@ exports.module = function* (url, source, attributes, artifacts, visited, opts =
     }
   }
 
-  visited.add(url.href)
-
   if (resolutions) {
     if (yield* exports.preresolved(url, source, resolutions, artifacts, visited, opts)) {
       return true
@@ -165,7 +182,8 @@ exports.module = function* (url, source, attributes, artifacts, visited, opts =
       imports['#package'] = packageURL.href
 
       yield {
-        children: exports.package(packageURL, source, artifacts, visited, opts)
+        children: exports.package(packageURL, source, artifacts, visited, opts),
+        deferred: false
       }
 
       break
@@ -185,89 +203,27 @@ exports.module = function* (url, source, attributes, artifacts, visited, opts =
     }
   }
 
-  let type = 0
+  const type = moduleType(url, attributes, info, opts)
 
-  if (typeof attributes.type === 'string') {
-    switch (attributes.type) {
-      case 'script':
-        type = constants.SCRIPT
-        break
-      case 'module':
-        type = constants.MODULE
-        break
-      case 'json':
-        type = constants.JSON
-        break
-      case 'bundle':
-        type = constants.BUNDLE
-        break
-      case 'addon':
-        type = constants.ADDON
-        break
-      case 'binary':
-        type = constants.BINARY
-        break
-      case 'text':
-        type = constants.TEXT
-        break
-    }
-  } else {
-    const match = url.pathname.match(/\.[a-z]+$/)
-
-    if (match !== null) {
-      let [extension] = match
-
-      if (aliases !== null && extension in aliases) extension = aliases[extension]
+  const lexer = { imports: [], exports: [] }
 
-      switch (extension) {
-        case '.js':
-        case '.ts': {
-          const isESM =
-            defaultType === constants.MODULE || (info !== null && info.type === 'module')
-
-          type = isESM ? constants.MODULE : constants.SCRIPT
-          break
-        }
-        case '.cjs':
-        case '.cts':
-          type = constants.SCRIPT
-          break
-        case '.mjs':
-        case '.mts':
-          type = constants.MODULE
-          break
-        case '.json':
-          type = constants.JSON
-          break
-        case '.bundle':
-          type = constants.BUNDLE
-          break
-        case '.bare':
-        case '.node':
-          type = constants.ADDON
-          break
-        case '.bin':
-          type = constants.BINARY
-          break
-        case '.txt':
-          type = constants.TEXT
-          break
-      }
+  // Assets are opaque; their contents are never interpreted, so we only recurse
+  // into modules that are actually imported as modules. An asset that happens
+  // to be a script, module, or addon is not lexed and its imports and addons
+  // are not followed.
+  if (asset === false) {
+    if (type === constants.SCRIPT || type === constants.MODULE) {
+      yield* exports.imports(url, source, imports, artifacts, lexer, visited, opts)
+    } else if (type === constants.ADDON) {
+      yield* exports.addons(url, artifacts, visited, opts)
     }
   }
 
-  const lexer = { imports: [] }
-
-  if (type === constants.SCRIPT || type === constants.MODULE) {
-    yield* exports.imports(url, source, imports, artifacts, lexer, visited, opts)
-  } else if (type === constants.ADDON) {
-    yield* exports.addons(url, artifacts, visited, opts)
-  }
-
   yield {
     dependency: {
       url: exports.alias(url, opts),
       source,
+      type,
       imports: compressImportsMap(imports),
       lexer
     }
@@ -279,22 +235,31 @@ exports.module = function* (url, source, attributes, artifacts, visited, opts =
 exports.package = function* (url, source, artifacts, visited, opts = {}) {
   if (visited.has(url.href)) return false
 
+  visited.add(url.href)
+
   if (source === null) {
     source = yield { module: url }
 
     if (source === null) return false
   }
 
-  visited.add(url.href)
-
   const info = JSON.parse(source)
 
   if (info) {
-    yield { dependency: { url, source, imports: {}, lexer: { imports: [] } } }
+    yield {
+      dependency: {
+        url,
+        source,
+        type: constants.JSON,
+        imports: {},
+        lexer: { imports: [], exports: [] }
+      }
+    }
 
     if (info.assets) {
       yield {
-        children: exports.assets(info.assets, url, artifacts, visited, opts)
+        children: exports.assets(info.assets, url, artifacts, visited, opts),
+        deferred: false
       }
     }
 
@@ -326,7 +291,8 @@ exports.preresolved = function* (url, source, resolutions, artifacts, visited, o
 
         if (specifier === '#package') {
           yield {
-            children: exports.package(url, null, artifacts, visited, opts)
+            children: exports.package(url, null, artifacts, visited, opts),
+            deferred: false
           }
         } else if (
           url.protocol !== builtinProtocol &&
@@ -334,7 +300,8 @@ exports.preresolved = function* (url, source, resolutions, artifacts, visited, o
           url.protocol !== deferredProtocol
         ) {
           yield {
-            children: exports.module(url, null, {}, artifacts, visited, opts)
+            children: exports.module(url, null, {}, artifacts, visited, opts),
+            deferred: false
           }
         }
       } else {
@@ -343,12 +310,24 @@ exports.preresolved = function* (url, source, resolutions, artifacts, visited, o
     }
   }
 
+  // The type is derived from the extension and default type alone; unlike
+  // `module`, the preresolved path does not read the package scope, so any
+  // `package.json` type declaration is not consulted here.
+  const type = moduleType(url, {}, null, opts)
+
+  const lexer = { imports: [], exports: [] }
+
+  if (type === constants.SCRIPT || type === constants.MODULE) {
+    lexer.exports = lex(source).exports
+  }
+
   yield {
     dependency: {
       url: exports.alias(url, opts),
       source,
+      type,
       imports: compressImportsMap(imports),
-      lexer: { imports: [] }
+      lexer
     }
   }
 
@@ -368,7 +347,11 @@ exports.imports = function* (parentURL, source, imports, artifacts, lexer, visit
 
   const queue = []
 
-  for (const entry of lex(source).imports) {
+  const lexed = lex(source)
+
+  lexer.exports = lexed.exports
+
+  for (const entry of lexed.imports) {
     let specifier = entry.specifier
     let condition = 'default'
 
@@ -377,6 +360,10 @@ exports.imports = function* (parentURL, source, imports, artifacts, lexer, visit
       condition = 'addon'
     } else if (entry.type & lex.constants.ASSET) {
       condition = 'asset'
+    } else if (entry.type & lex.constants.REQUIRE) {
+      condition = 'require'
+    } else if (entry.type & lex.constants.IMPORT) {
+      condition = 'import'
     }
 
     if (entry.attributes.imports) {
@@ -436,7 +423,11 @@ exports.imports = function* (parentURL, source, imports, artifacts, lexer, visit
 
           for (const url of yield { prefix }) {
             yield {
-              children: exports.module(url, null, {}, artifacts, visited, opts)
+              children: exports.module(url, null, {}, artifacts, visited, {
+                ...opts,
+                asset: true
+              }),
+              deferred: true
             }
 
             addURL(artifacts.assets, url)
@@ -458,7 +449,8 @@ exports.imports = function* (parentURL, source, imports, artifacts, lexer, visit
             }
 
             yield {
-              children: exports.module(url, source, attributes, artifacts, visited, opts)
+              children: exports.module(url, source, attributes, artifacts, visited, opts),
+              deferred: false
             }
 
             resolved = yielded = true
@@ -478,7 +470,7 @@ exports.imports = function* (parentURL, source, imports, artifacts, lexer, visit
     matchedConditions.pop()
 
     if (resolutions === 0) {
-      let message = `Cannot find ${condition === 'default' ? 'module' : condition} '${specifier}' imported from '${parentURL.href}'`
+      let message = `Cannot find ${condition === 'addon' || condition === 'asset' ? condition : 'module'} '${specifier}' imported from '${parentURL.href}'`
 
       if (candidates.length > 0) {
         message += '\nCandidates:'
@@ -511,7 +503,8 @@ exports.addons = function* (parentURL, artifacts, visited, opts = {}) {
 
     for (const url of yield { prefix }) {
       yield {
-        children: exports.module(url, null, {}, artifacts, visited, opts)
+        children: exports.module(url, null, {}, artifacts, visited, opts),
+        deferred: false
       }
 
       addURL(artifacts.addons, url)
@@ -535,7 +528,11 @@ exports.assets = function* (patterns, parentURL, artifacts, visited, opts = {})
       addURL(artifacts.assets, url)
 
       yield {
-        children: exports.module(url, source, {}, artifacts, visited, opts)
+        children: exports.module(url, source, {}, artifacts, visited, {
+          ...opts,
+          asset: true
+        }),
+        deferred: true
       }
 
       yielded = true
@@ -608,6 +605,70 @@ exports.patternMatches = function* patternMatches(pattern, parentURL, matches, o
   return matches
 }
 
+// Determine the module type from an explicit type attribute, if given, and
+// otherwise from the module's extension. For the ambiguous `.js` and `.ts`
+// extensions the nearest `package.json`, passed as `info`, and the default type
+// break the tie. Returns `0` when the type cannot be determined.
+function moduleType(url, attributes, info, opts = {}) {
+  const { defaultType = constants.SCRIPT, aliases = null } = opts
+
+  if (typeof attributes.type === 'string') {
+    switch (attributes.type) {
+      case 'script':
+        return constants.SCRIPT
+      case 'module':
+        return constants.MODULE
+      case 'json':
+        return constants.JSON
+      case 'bundle':
+        return constants.BUNDLE
+      case 'addon':
+        return constants.ADDON
+      case 'binary':
+        return constants.BINARY
+      case 'text':
+        return constants.TEXT
+    }
+
+    return 0
+  }
+
+  const match = url.pathname.match(/\.[a-z]+$/)
+
+  if (match === null) return 0
+
+  let [extension] = match
+
+  if (aliases !== null && extension in aliases) extension = aliases[extension]
+
+  switch (extension) {
+    case '.js':
+    case '.ts':
+      return defaultType === constants.MODULE || (info !== null && info.type === 'module')
+        ? constants.MODULE
+        : constants.SCRIPT
+    case '.cjs':
+    case '.cts':
+      return constants.SCRIPT
+    case '.mjs':
+    case '.mts':
+      return constants.MODULE
+    case '.json':
+      return constants.JSON
+    case '.bundle':
+      return constants.BUNDLE
+    case '.bare':
+    case '.node':
+      return constants.ADDON
+    case '.bin':
+      return constants.BINARY
+    case '.txt':
+      return constants.TEXT
+  }
+
+  return 0
+}
+
 function addURL(collection, url) {
   if (Array.isArray(collection)) {
     let lo = 0
@@ -708,6 +769,12 @@ function compressImportsMapEntry(resolved) {
     if (entry[0] === 'default') primary = entry[1]
   }
 
+  if (entries.length === 0) return resolved
+
+  const [, first] = entries[0]
+
+  if (entries.every(([, resolved]) => resolved === first)) return first
+
   entries = entries.filter(
     ([condition, resolved]) => condition === 'default' || resolved !== primary
   )

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bare-module-traverse",
-  "version": "2.2.0",
+  "version": "2.3.1",
   "description": "Low-level module graph traversal for Bare",
   "exports": {
     ".": {