@davidwells/extract-deps 0.1.13 → 0.1.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@davidwells/extract-deps",
-  "version": "0.1.13",
+  "version": "0.1.14",
   "description": "Get imports and requires from a codebase without AST",
   "main": "src/index.js",
   "exports": {

package/src/dep-graph.js

@@ -90,7 +90,10 @@ function fsExistsSync(path) {
 
 function isFileSync(filePath) {
   try {
-    return fs.statSync(filePath).isFile()
+    // Most probes miss (each import tries several extensions): a missing path returns
+    // undefined instead of building and throwing an ENOENT error.
+    const stat = fs.statSync(filePath, { throwIfNoEntry: false })
+    return stat ? stat.isFile() : false
   } catch (error) {
     return false
   }
@@ -448,7 +451,149 @@ function depGraph(filePath, options = {}) {
 
 
 /**
- * Get all dependency file paths in build order (dependencies first), deduplicated
+ * Order every file in the graph so dependencies come before dependents, each file once.
+ *
+ * Kahn's algorithm over the strongly connected components (Tarjan) of the file graph:
+ * an import cycle (or self-import) is one component, emitted as a unit with its members
+ * in discovery order. Duplicate edges (one file importing a module twice) count once.
+ * For an acyclic graph every component is a single file, and the queue seeding and
+ * dependent visiting orders match a plain per-file Kahn's sort over the graph's
+ * insertion order. O(V + E).
+ *
+ * @param {Map<string, string[]>} graph - file -> deps (file paths and package names)
+ * @returns {string[]}
+ */
+function buildOrder(graph) {
+  const nodes = Array.from(graph.keys())
+  const n = nodes.length
+  /** @type {Map<string, number>} */
+  const indexOf = new Map()
+  for (let i = 0; i < n; i++) indexOf.set(nodes[i], i)
+
+  // Unique file deps per node (first-occurrence order) and reverse edges. Dependents
+  // are appended while walking nodes in graph order, so each reverse list is in graph order.
+  /** @type {number[][]} */
+  const deps = new Array(n)
+  /** @type {number[][]} */
+  const dependents = new Array(n)
+  for (let i = 0; i < n; i++) dependents[i] = []
+  const seenStamp = new Int32Array(n).fill(-1)
+  for (let i = 0; i < n; i++) {
+    const out = []
+    for (const dep of graph.get(nodes[i])) {
+      const j = indexOf.get(dep) // package names (and unvisited paths) are not nodes
+      if (j === undefined || seenStamp[j] === i) continue
+      seenStamp[j] = i
+      out.push(j)
+      dependents[j].push(i)
+    }
+    deps[i] = out
+  }
+
+  // Tarjan's SCC, iterative (deep import chains must not overflow the call stack)
+  const UNVISITED = -1
+  const order = new Int32Array(n).fill(UNVISITED)
+  const low = new Int32Array(n)
+  const onStack = new Uint8Array(n)
+  const comp = new Int32Array(n)
+  const sccStack = []
+  const callStack = [] // [node, next dep position]
+  let counter = 0
+  let compCount = 0
+  for (let root = 0; root < n; root++) {
+    if (order[root] !== UNVISITED) continue
+    callStack.push(root, 0)
+    order[root] = low[root] = counter++
+    sccStack.push(root)
+    onStack[root] = 1
+    while (callStack.length > 0) {
+      const top = callStack.length - 2
+      const v = callStack[top]
+      const edges = deps[v]
+      const pos = callStack[top + 1]
+      if (pos < edges.length) {
+        callStack[top + 1] = pos + 1
+        const w = edges[pos]
+        if (order[w] === UNVISITED) {
+          order[w] = low[w] = counter++
+          sccStack.push(w)
+          onStack[w] = 1
+          callStack.push(w, 0)
+        } else if (onStack[w] && order[w] < low[v]) {
+          low[v] = order[w]
+        }
+        continue
+      }
+      callStack.length = top
+      if (top > 0) {
+        const parent = callStack[top - 2]
+        if (low[v] < low[parent]) low[parent] = low[v]
+      }
+      if (low[v] === order[v]) {
+        let w
+        do {
+          w = sccStack.pop()
+          onStack[w] = 0
+          comp[w] = compCount
+        } while (w !== v)
+        compCount++
+      }
+    }
+  }
+
+  // Members per component in graph (discovery) order
+  /** @type {number[][]} */
+  const members = new Array(compCount)
+  for (let c = 0; c < compCount; c++) members[c] = []
+  for (let i = 0; i < n; i++) members[comp[i]].push(i)
+
+  // Component in-degree: distinct other components it depends on
+  const compInDegree = new Int32Array(compCount)
+  const compStamp = new Int32Array(compCount).fill(-1)
+  for (let c = 0; c < compCount; c++) {
+    for (const v of members[c]) {
+      for (const w of deps[v]) {
+        const cw = comp[w]
+        if (cw === c || compStamp[cw] === c) continue
+        compStamp[cw] = c
+        compInDegree[c]++
+      }
+    }
+  }
+
+  // Kahn's over components, seeded in graph order
+  const queue = []
+  const queued = new Uint8Array(compCount)
+  for (let i = 0; i < n; i++) {
+    const c = comp[i]
+    if (compInDegree[c] === 0 && !queued[c]) {
+      queued[c] = 1
+      queue.push(c)
+    }
+  }
+  const result = []
+  compStamp.fill(-1)
+  for (let head = 0; head < queue.length; head++) {
+    const c = queue[head]
+    for (const v of members[c]) result.push(nodes[v])
+    for (const v of members[c]) {
+      for (const d of dependents[v]) {
+        const cd = comp[d]
+        if (cd === c || compStamp[cd] === c) continue
+        compStamp[cd] = c
+        if (--compInDegree[cd] === 0) {
+          queued[cd] = 1
+          queue.push(cd)
+        }
+      }
+    }
+  }
+  return result
+}
+
+/**
+ * Get all dependency file paths in build order (dependencies first), deduplicated.
+ * Files in an import cycle are kept together, in discovery order.
  * @param {string} filePath - Entry file path
  * @param {object} options - Options to pass to depGraph
  * @param {boolean} [options.verifyGraph] - If true, verify imports/exports. Default: true
@@ -492,42 +637,8 @@ function resolveDepPaths(filePath, options = {}) {
     }
   }
 
-  // Calculate in-degree (number of file dependencies) for each file
-  const inDegree = new Map()
-  for (const [file, deps] of graph) {
-    // Only count file deps, not node_modules
-    const fileDeps = deps.filter(d => d.startsWith('/'))
-    inDegree.set(file, fileDeps.length)
-  }
-
-  // Queue of files with no dependencies (can be processed first)
-  const queue = []
-  for (const [file, degree] of inDegree) {
-    if (degree === 0) {
-      queue.push(file)
-    }
-  }
-
-  const buildOrder = []
-
-  while (queue.length > 0) {
-    const file = queue.shift()
-    buildOrder.push(file)
-
-    // Find all files that depend on this file and decrement their in-degree
-    for (const [dependent, deps] of graph) {
-      if (deps.includes(file)) {
-        const newDegree = inDegree.get(dependent) - 1
-        inDegree.set(dependent, newDegree)
-        if (newDegree === 0) {
-          queue.push(dependent)
-        }
-      }
-    }
-  }
-
   const result = {
-    files: buildOrder,
+    files: buildOrder(graph),
     nodeModules: Array.from(nodeModulesSet),
     builtIns: Array.from(builtInsSet)
   }

package/src/get-cjs-exports.js

@@ -34,7 +34,14 @@ function stripComments(code) {
   return result
 }
 
+// Every scanner below is gated on one of these in the sanitized text. Sanitizing only
+// blanks characters to spaces (length-preserving), so a needle without whitespace that
+// isn't in the source can't be in the sanitized text either.
+const CJS_NEEDLES = ['exports', 'Object.assign', 'Object.defineProperty', 'Object.keys(', '__exportStar', '__reExport']
+
 function getCjsExports(code) {
+  // Most ESM/TS files mention none of them: skip the sanitize pass entirely.
+  if (!CJS_NEEDLES.some(needle => code.includes(needle))) return []
   // Single tokenizer-based pass that blanks comments and code-bearing
   // strings/templates together. Length is preserved so the position-based
   // logic below (which slices into the original `code` for full export

package/src/get-exports.js

@@ -1,4 +1,4 @@
-const { getNames, extractExportName, stripComments, stripStringContents } = require('./utils')
+const { getNames, extractExportName, stripCommentsThenStrings } = require('./utils')
 
 // https://regex101.com/r/example/1
 const EXPORT_DEFAULT_REGEX = /export\s+default\s+(?:(?:async\s+)?function\s+([\w$]+)|class\s+([\w$]+)|([\w$]+|\([^)]*\)\s*=>|{[\s\S]*?}|\[[\s\S]*?\]|'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"|`(?:[^`\\]|\\.)*`|[\w$]+(?:\s*[+\-*/%]\s*[\w$]+)*))/gm
@@ -22,8 +22,7 @@ function getExports(code = '', options = {}) {
   // alone would mis-pair an apostrophe inside a JSDoc comment (`Netlify's …`)
   // with a later quote and shift subsequent backtick pairing, which could
   // swallow real `export` declarations a few lines later.
-  code = stripComments(code)
-  code = stripStringContents(code)
+  code = stripCommentsThenStrings(code)
   const defaultExports = getDefaultExports(code)
   const namedExports = getNamedExports(code)
   const namedDeclarationExports = getNamedDeclarationExports(code)

package/src/get-imports.js

@@ -1,4 +1,4 @@
-const { isBuiltIn, isRelativeImport, isRelativeImportWithQuotes, getNames, stripComments, stripStringContents } = require('./utils')
+const { isBuiltIn, isRelativeImport, isRelativeImportWithQuotes, getNames, stripCommentsThenStrings } = require('./utils')
 
 // https://regex101.com/r/cfEmlm/1
 // Path excludes quotes, newlines, and `;` so multi-statement lines like
@@ -43,10 +43,11 @@ function getImports(code = '', options = {}) {
   // removed before `stripStringContents` looks for string literals.
   // (Running the two in the opposite order let comment chars masquerade as
   // string openers and swallow real `import`/`export` lines further down.)
-  code = stripComments(code)
-  code = stripStringContents(code)
+  code = stripCommentsThenStrings(code)
   const staticImports = getStaticImports(code)
-  const dynamicImports = getDynamicImports(code)
+  // Every dynamic-import match contains `import(` (most files have none; the regex's
+  // optional declaration prefix makes a miss scan the whole text).
+  const dynamicImports = code.includes('import(') ? getDynamicImports(code) : []
   if (options.includeAssertionImports) {
     const assertionImports = getAssertionImports(code)
     return staticImports.concat(dynamicImports).concat(assertionImports).sort((a, b) => a.index - b.index)

package/src/get-requires.js

@@ -1,4 +1,4 @@
-const { isBuiltIn, isRelativeImport, getNames, stripComments, stripStringContents } = require('./utils')
+const { isBuiltIn, isRelativeImport, getNames, stripCommentsThenStrings } = require('./utils')
 
 // Updated regex to exclude ES6 dynamic imports
 // require() must always be present — previous version made it optional, causing false matches
@@ -111,8 +111,7 @@ function getRequires(code) {
   // alone would mis-eat `//` inside strings; running it first instead means
   // comment chars (incl. apostrophes/backticks inside JSDoc) disappear
   // before `stripStringContents` looks for real string literals.
-  code = stripComments(code)
-  code = stripStringContents(code)
+  code = stripCommentsThenStrings(code)
   const staticRequires = getStaticRequires(code)
   const dynamicRequires = getDynamicRequires(code)
   return staticRequires.concat(dynamicRequires).sort((a, b) => a.index - b.index)

package/src/utils.js

@@ -291,6 +291,26 @@ function stripComments(code) {
   return stripCommentsImpl(code)
 }
 
+// Last input/output of stripCommentsThenStrings: one parse runs the import,
+// require and export scanners over the same source text in a row.
+let lastStripInput = null
+let lastStripOutput = ''
+
+/**
+ * `stripStringContents(stripComments(code))`, the prelude of the import, require
+ * and export scanners (comments first, then strings: see getImports). Pure in
+ * `code`; the latest result is kept for the next scanner of the same text.
+ * @param {string} code
+ * @returns {string}
+ */
+function stripCommentsThenStrings(code) {
+  if (code === lastStripInput) return lastStripOutput
+  const output = stripStringContents(stripCommentsImpl(code))
+  lastStripInput = code
+  lastStripOutput = output
+  return output
+}
+
 /**
  * Check if a module specifier is a Node.js built-in
  * @param {string} moduleName - The module specifier (e.g., 'fs', 'node:fs/promises')
@@ -504,6 +524,7 @@ module.exports = {
   stripComments,
   stripStringContents,
   stripCommentsAndStrings,
+  stripCommentsThenStrings,
   isBuiltIn,
   isRelativeImport,
   isRelativeImportWithQuotes,

package/tests/dep-graph-cycles.test.js

@@ -0,0 +1,111 @@
+// resolveDepPaths must return every reachable file exactly once, including
+// import cycles, files that depend on a cycle, self-imports and files that
+// import the same module more than once (duplicate edges).
+const { test } = require('uvu')
+const assert = require('uvu/assert')
+const path = require('path')
+const { depGraph, resolveDepPaths } = require('../src/dep-graph.js')
+
+const FIXTURES = path.resolve(__dirname, './fixtures/cycle-fixtures')
+
+/** @param {string} rel */
+const fixture = (rel) => path.join(FIXTURES, rel)
+
+/**
+ * Resolve deps and return basenames (fixtures use unique names per directory)
+ * @param {string} entry
+ */
+function names(entry) {
+  const result = resolveDepPaths(entry, { returnErrors: true })
+  assert.equal(result.errors, [], 'fixture should resolve without errors')
+  return result.files.map(f => path.basename(f))
+}
+
+/**
+ * Every file in the graph (= everything reachable from the entry) appears exactly once
+ * @param {string} entry
+ */
+function assertComplete(entry) {
+  const graph = depGraph(entry)
+  const { files } = resolveDepPaths(entry)
+  assert.is(new Set(files).size, files.length, 'no duplicates')
+  assert.equal([...files].sort(), [...graph.keys()].sort(), 'files equal the reachable set')
+}
+
+test('2-file cycle: both cycle members and their dependent are returned', () => {
+  const entry = fixture('two-cycle/index.js')
+  assertComplete(entry)
+  assert.equal(names(entry), ['a.js', 'b.js', 'index.js'])
+})
+
+test('cycle through the entry file: entry and its cycle peer are returned', () => {
+  const entry = fixture('cycle-entry/index.js')
+  assertComplete(entry)
+  assert.equal(names(entry), ['leaf.js', 'index.js', 'peer.js'])
+})
+
+test('3-file cycle with a dependent outside it and a leaf below it', () => {
+  const entry = fixture('three-cycle/index.js')
+  assertComplete(entry)
+  // leaf first (dependency of the cycle), cycle members in discovery order,
+  // then w (depends on the cycle), then the entry
+  assert.equal(names(entry), ['leaf.js', 'x.js', 'y.js', 'z.js', 'w.js', 'index.js'])
+})
+
+test('self-import does not drop the file or its dependents', () => {
+  const entry = fixture('self-import/index.js')
+  assertComplete(entry)
+  assert.equal(names(entry), ['leaf.js', 'self.js', 'index.js'])
+})
+
+test('same module required twice (same and different specifiers)', () => {
+  const entry = fixture('duplicate-edge/index.js')
+  assertComplete(entry)
+  assert.equal(names(entry), ['leaf.js', 'other.js', 'util.js', 'index.js'])
+})
+
+test('same module imported (ESM) and required (CJS) in one file', () => {
+  const entry = fixture('mixed-duplicate/index.mjs')
+  assertComplete(entry)
+  assert.equal(names(entry), ['base.mjs', 'helper.mjs', 'index.mjs'])
+})
+
+test('diamond (acyclic): shared dependency once, dependencies before dependents', () => {
+  const entry = fixture('diamond/index.js')
+  assertComplete(entry)
+  assert.equal(names(entry), ['d.js', 'c.js', 'a.js', 'b.js', 'index.js'])
+})
+
+test('acyclic graphs keep dependencies-first order', () => {
+  for (const rel of ['diamond/index.js', 'duplicate-edge/index.js', 'mixed-duplicate/index.mjs']) {
+    const entry = fixture(rel)
+    const graph = depGraph(entry)
+    const { files } = resolveDepPaths(entry)
+    const pos = new Map(files.map((f, i) => [f, i]))
+    for (const [file, deps] of graph) {
+      for (const dep of deps) {
+        assert.ok(pos.get(dep) < pos.get(file), `${path.basename(dep)} before ${path.basename(file)} in ${rel}`)
+      }
+    }
+  }
+})
+
+test('files outside a cycle still come after everything they depend on', () => {
+  const entry = fixture('three-cycle/index.js')
+  const { files } = resolveDepPaths(entry)
+  const pos = new Map(files.map(f => [path.basename(f), files.indexOf(f)]))
+  for (const member of ['x.js', 'y.js', 'z.js']) {
+    assert.ok(pos.get(member) < pos.get('w.js'), `${member} before w.js`)
+    assert.ok(pos.get(member) > pos.get('leaf.js'), `leaf.js before ${member}`)
+  }
+  assert.is(files[files.length - 1], entry, 'entry last when it is not in a cycle')
+})
+
+test('output is deterministic across calls', () => {
+  for (const rel of ['two-cycle/index.js', 'three-cycle/index.js', 'cycle-entry/index.js', 'self-import/index.js']) {
+    const entry = fixture(rel)
+    assert.equal(resolveDepPaths(entry).files, resolveDepPaths(entry).files)
+  }
+})
+
+test.run()

package/tests/fixtures/cycle-fixtures/cycle-entry/index.js

@@ -0,0 +1,2 @@
+const peer = require('./peer')
+module.exports = { run: () => peer }

package/tests/fixtures/cycle-fixtures/cycle-entry/leaf.js

@@ -0,0 +1 @@
+module.exports = { leaf: true }

package/tests/fixtures/cycle-fixtures/cycle-entry/peer.js

@@ -0,0 +1,3 @@
+const index = require('./index')
+const leaf = require('./leaf')
+module.exports = { index, leaf }

package/tests/fixtures/cycle-fixtures/diamond/a.js

@@ -0,0 +1,2 @@
+const c = require('./c')
+module.exports = { a: () => c }

package/tests/fixtures/cycle-fixtures/diamond/b.js

@@ -0,0 +1,2 @@
+const c = require('./c')
+module.exports = { b: () => c }

package/tests/fixtures/cycle-fixtures/diamond/c.js

@@ -0,0 +1,2 @@
+const d = require('./d')
+module.exports = { c: () => d }

package/tests/fixtures/cycle-fixtures/diamond/d.js

@@ -0,0 +1 @@
+module.exports = { d: true }

package/tests/fixtures/cycle-fixtures/diamond/index.js

@@ -0,0 +1,3 @@
+const a = require('./a')
+const b = require('./b')
+module.exports = { a, b }

package/tests/fixtures/cycle-fixtures/duplicate-edge/index.js

@@ -0,0 +1,5 @@
+const a = require('./util')
+const b = require('./util.js')
+const c = require('./util')
+const other = require('./other')
+module.exports = { a, b, c, other }

package/tests/fixtures/cycle-fixtures/duplicate-edge/leaf.js

@@ -0,0 +1 @@
+module.exports = { leaf: true }

package/tests/fixtures/cycle-fixtures/duplicate-edge/other.js

@@ -0,0 +1 @@
+module.exports = { other: true }

package/tests/fixtures/cycle-fixtures/duplicate-edge/util.js

@@ -0,0 +1,2 @@
+const leaf = require('./leaf')
+module.exports = { util: () => leaf }

package/tests/fixtures/cycle-fixtures/mixed-duplicate/base.mjs

@@ -0,0 +1 @@
+export const base = 1

package/tests/fixtures/cycle-fixtures/mixed-duplicate/helper.mjs

@@ -0,0 +1,2 @@
+import { base } from './base.mjs'
+export const helper = () => base

package/tests/fixtures/cycle-fixtures/mixed-duplicate/index.mjs

@@ -0,0 +1,3 @@
+import { helper } from './helper.mjs'
+const again = require('./helper.mjs')
+export const run = () => [helper, again]

package/tests/fixtures/cycle-fixtures/self-import/index.js

@@ -0,0 +1,2 @@
+const self = require('./self')
+module.exports = { self }

package/tests/fixtures/cycle-fixtures/self-import/leaf.js

@@ -0,0 +1 @@
+module.exports = { leaf: true }

package/tests/fixtures/cycle-fixtures/self-import/self.js

@@ -0,0 +1,3 @@
+const me = require('./self')
+const leaf = require('./leaf')
+module.exports = { me, leaf }

package/tests/fixtures/cycle-fixtures/three-cycle/index.js

@@ -0,0 +1,3 @@
+const w = require('./w')
+const x = require('./x')
+module.exports = { w, x }

package/tests/fixtures/cycle-fixtures/three-cycle/leaf.js

@@ -0,0 +1 @@
+module.exports = { leaf: true }

package/tests/fixtures/cycle-fixtures/three-cycle/w.js

@@ -0,0 +1,2 @@
+const x = require('./x')
+module.exports = { w: () => x }

package/tests/fixtures/cycle-fixtures/three-cycle/x.js

@@ -0,0 +1,2 @@
+const y = require('./y')
+module.exports = { x: () => y }

package/tests/fixtures/cycle-fixtures/three-cycle/y.js

@@ -0,0 +1,2 @@
+const z = require('./z')
+module.exports = { y: () => z }

package/tests/fixtures/cycle-fixtures/three-cycle/z.js

@@ -0,0 +1,3 @@
+const x = require('./x')
+const leaf = require('./leaf')
+module.exports = { z: () => [x, leaf] }

package/tests/fixtures/cycle-fixtures/two-cycle/a.js

@@ -0,0 +1,2 @@
+const b = require('./b')
+module.exports = { ping: () => b.pong() }

package/tests/fixtures/cycle-fixtures/two-cycle/b.js

@@ -0,0 +1,2 @@
+const a = require('./a')
+module.exports = { pong: () => a }

package/tests/fixtures/cycle-fixtures/two-cycle/index.js

@@ -0,0 +1,2 @@
+const a = require('./a')
+module.exports = { run: () => a.ping() }