@cdxgen/cdxgen 13.2.0 → 13.4.0

package/lib/inventory/aiCollector.js

@@ -90,8 +90,11 @@ const isVendoredArtifactPath = (filePath) => {
 };
 const GGUF_MIN_ALIGNMENT = 4;
 const GGUF_MAX_ALIGNMENT = 1024 * 1024;
+// Tested against full paths. The config exceptions follow IGNORE_FILE_PATTERN
+// in analyzer.js, with the app exception anchored to a path separator instead
+// of the start of a file name.
 const IGNORE_SOURCE_FILE_PATTERN =
-  /(^|\/|\\)(__tests__|fixtures?|examples?|samples?)($|\/|\\)|(^|\/|\\)(test|spec|mock|setup-jest|conftest|sitecustomize)\.(js|ts|tsx|py)$|(?<!vite\.|vue\.)(conf|config)\.(js|ts|tsx)$/iu;
+  /(^|\/|\\)(__tests__|fixtures?|examples?|samples?)($|\/|\\)|(^|\/|\\)(test|spec|mock|setup-jest|conftest|sitecustomize)\.([cm]?[jt]sx?|py)$|(?<!vite\.|vue\.|svelte\.|astro\.|(?:^|\/|\\)app\.)(conf|config)\.[cm]?[jt]sx?$/iu;
 const HOST_PROVIDER_PATTERNS = [
   ["openai", /(?:^|\.)openai\.com$/iu],
   ["anthropic", /(?:^|\.)anthropic\.com$/iu],

package/lib/inventory/analyzer.js

@@ -68,9 +68,19 @@ const IGNORE_DIRS = readEnvironmentVariable("ASTGEN_IGNORE_DIRS")
       "coverage",
     ];
 
+// The pattern is tested against file names. Config files are skipped because
+// their imports are mostly build and test tooling. The exceptions hold code the
+// application is built from: the Vite, Vue CLI, Svelte, and Astro configs name
+// the build plugins and integrations the app needs, and app.config.* holds
+// application code, such as the root providers of an Angular app. The app
+// exception is anchored to the start of the name, so names such as
+// webapp.config.ts are still skipped. [cm]?[jt]sx? covers every JavaScript and
+// TypeScript module extension, as the default test globs of Jest and Vitest do.
+// Keep the config exceptions in step with IGNORE_SOURCE_FILE_PATTERN in
+// aiCollector.js.
 const IGNORE_FILE_PATTERN = new RegExp(
   readEnvironmentVariable("ASTGEN_IGNORE_FILE_PATTERN") ||
-    "(test|spec|mock|setup-jest|\\.d)\\.(js|ts|tsx)$|(?<!vite\\.|vue\\.|svelte\\.)(conf|config)\\.(js|ts|tsx)$",
+    "(test|spec|mock|setup-jest|\\.d)\\.[cm]?[jt]sx?$|(?<!vite\\.|vue\\.|svelte\\.|astro\\.|^app\\.)(conf|config)\\.[cm]?[jt]sx?$",
   "i",
 );
 
@@ -210,6 +220,46 @@ const getAllFiles = (
   return result;
 };
 
+/**
+ * Walk a directory tree once and return the files whose names end with any of
+ * the given extensions.
+ *
+ * Files are grouped by extension in the order of `extensions`, and each group
+ * keeps the walk order. That is the order one walk per extension produced, and
+ * occurrences reach the BOM in it.
+ *
+ * @param {string} dir Directory to walk
+ * @param {string[]} extensions File extensions, each with its leading dot
+ * @param {boolean|Object} deepOrOptions Deep flag, or `{ deep, exclude, noIgnore }`
+ * @returns {string[]} Matching file paths
+ */
+const getAllFilesByExtensions = (dir, extensions, deepOrOptions) => {
+  const searchOptions = normalizeAnalyzerSearchOptions(deepOrOptions);
+  const files = getAllFiles(
+    searchOptions.deep,
+    dir,
+    undefined,
+    undefined,
+    undefined,
+    new RegExp(
+      `(?:${extensions.map((extension) => `\\${extension}`).join("|")})$`,
+    ),
+    dir,
+    searchOptions.exclude,
+    searchOptions.noIgnore,
+  );
+  const filesByExtension = extensions.map(() => []);
+  for (const file of files) {
+    const extensionIndex = extensions.findIndex((extension) =>
+      file.endsWith(extension),
+    );
+    if (extensionIndex !== -1) {
+      filesByExtension[extensionIndex].push(file);
+    }
+  }
+  return filesByExtension.flat();
+};
+
 const babelParserOptions = {
   sourceType: "unambiguous",
   allowImportExportEverywhere: true,
@@ -1752,37 +1802,29 @@ export const parseFileASTTreeCollected = (src, file) => {
   return { imports, exports };
 };
 
+const JS_TS_SOURCE_EXTENSIONS = [
+  ".js",
+  ".jsx",
+  ".cjs",
+  ".mjs",
+  ".ts",
+  ".tsx",
+  ".mts",
+  ".cts",
+  ".vue",
+  ".svelte",
+];
+
 /**
- * Return paths to all (j|tsx?) files.
+ * Return paths to all JavaScript, TypeScript, Vue, and Svelte source files,
+ * found in one walk of the tree.
+ *
+ * @param {string} src Directory to walk
+ * @param {boolean|Object} deep Deep flag, or `{ deep, exclude, noIgnore }`
+ * @returns {string[]} Source file paths, grouped by extension
  */
 const getAllSrcJSAndTSFiles = (src, deep) =>
-  Promise.all(
-    [
-      ".js",
-      ".jsx",
-      ".cjs",
-      ".mjs",
-      ".ts",
-      ".tsx",
-      ".mts",
-      ".cts",
-      ".vue",
-      ".svelte",
-    ].map((extension) => {
-      const searchOptions = normalizeAnalyzerSearchOptions(deep);
-      return getAllFiles(
-        searchOptions.deep,
-        src,
-        extension,
-        undefined,
-        undefined,
-        undefined,
-        src,
-        searchOptions.exclude,
-        searchOptions.noIgnore,
-      );
-    }),
-  );
+  getAllFilesByExtensions(src, JS_TS_SOURCE_EXTENSIONS, deep);
 
 const getAngularWorkspaceManifestFiles = (
   dir,
@@ -2139,51 +2181,42 @@ const parseAngularStyleFiles = (
   excludePatterns,
   noIgnore = false,
 ) => {
-  for (const extension of ANGULAR_STYLE_FILE_EXTENSIONS) {
-    const styleFiles = getAllFiles(
-      deep,
-      src,
-      extension,
-      undefined,
-      undefined,
-      undefined,
-      src,
-      excludePatterns,
-      noIgnore,
-    );
-    for (const styleFile of styleFiles) {
-      let content;
-      try {
-        content = readFileSync(styleFile, "utf-8");
-      } catch {
-        continue;
-      }
-      for (const referencePattern of STYLE_PACKAGE_REFERENCE_PATTERNS) {
-        for (const match of content.matchAll(referencePattern)) {
-          const reference = match[1];
-          if (!reference) {
-            continue;
-          }
-          if (
-            match[0]?.trim().startsWith("url") &&
-            !isPackageStyleUrlReference(reference)
-          ) {
-            continue;
-          }
-          const prefix = content.slice(0, match.index || 0);
-          const lineNumber = prefix.split(/\r?\n/).length;
-          const lineStart = prefix.lastIndexOf("\n");
-          const columnNumber =
-            lineStart >= 0 ? prefix.length - lineStart - 1 : 0;
-          setAngularPackageRef(
-            allImports,
-            allExports,
-            src,
-            styleFile,
-            reference,
-            { column: columnNumber, line: lineNumber },
-          );
+  const styleFiles = getAllFilesByExtensions(
+    src,
+    ANGULAR_STYLE_FILE_EXTENSIONS,
+    { deep, exclude: excludePatterns, noIgnore },
+  );
+  for (const styleFile of styleFiles) {
+    let content;
+    try {
+      content = readFileSync(styleFile, "utf-8");
+    } catch {
+      continue;
+    }
+    for (const referencePattern of STYLE_PACKAGE_REFERENCE_PATTERNS) {
+      for (const match of content.matchAll(referencePattern)) {
+        const reference = match[1];
+        if (!reference) {
+          continue;
         }
+        if (
+          match[0]?.trim().startsWith("url") &&
+          !isPackageStyleUrlReference(reference)
+        ) {
+          continue;
+        }
+        const prefix = content.slice(0, match.index || 0);
+        const lineNumber = prefix.split(/\r?\n/).length;
+        const lineStart = prefix.lastIndexOf("\n");
+        const columnNumber = lineStart >= 0 ? prefix.length - lineStart - 1 : 0;
+        setAngularPackageRef(
+          allImports,
+          allExports,
+          src,
+          styleFile,
+          reference,
+          { column: columnNumber, line: lineNumber },
+        );
       }
     }
   }
@@ -3478,8 +3511,7 @@ export const findJSImportsExports = async (src, deep) => {
     // scope of astWorker.js. Absence is silent — tier 1 covers all scope
     // evidence.
     await primeSvelteParser();
-    const promiseMap = await getAllSrcJSAndTSFiles(src, deep);
-    const srcFiles = promiseMap.flat();
+    const srcFiles = getAllSrcJSAndTSFiles(src, deep);
     const tasks = srcFiles.map((file) => ({ src, file }));
     const parallelWorthwhile =
       totalSourceBytes(srcFiles) >= MIN_PARALLEL_PARSE_BYTES;
@@ -4580,8 +4612,7 @@ export const analyzeJsCryptoFile = (filePath) => {
 export const detectJsCryptoInventory = async (src, deep = false) => {
   let srcFiles = [];
   try {
-    const promiseMap = await getAllSrcJSAndTSFiles(src, deep);
-    srcFiles = promiseMap.flat();
+    srcFiles = getAllSrcJSAndTSFiles(src, deep);
   } catch {
     return { algorithms: [], libraries: [] };
   }
@@ -4626,31 +4657,7 @@ export const detectExtensionCapabilities = (src, deep = false) => {
   }
   let srcFiles = [];
   try {
-    const searchOptions = normalizeAnalyzerSearchOptions(deep);
-    srcFiles = [
-      ".js",
-      ".jsx",
-      ".cjs",
-      ".mjs",
-      ".ts",
-      ".tsx",
-      ".mts",
-      ".cts",
-      ".vue",
-      ".svelte",
-    ].flatMap((extension) =>
-      getAllFiles(
-        searchOptions.deep,
-        src,
-        extension,
-        undefined,
-        undefined,
-        undefined,
-        src,
-        searchOptions.exclude,
-        searchOptions.noIgnore,
-      ),
-    );
+    srcFiles = getAllSrcJSAndTSFiles(src, deep);
   } catch (_err) {
     return { capabilities: [], indicators: {} };
   }
@@ -5857,31 +5864,7 @@ export const detectMcpInventory = (src, deep = false) => {
   const servicesByKey = new Map();
   let srcFiles = [];
   try {
-    const searchOptions = normalizeAnalyzerSearchOptions(deep);
-    srcFiles = [
-      ".js",
-      ".jsx",
-      ".cjs",
-      ".mjs",
-      ".ts",
-      ".tsx",
-      ".mts",
-      ".cts",
-      ".vue",
-      ".svelte",
-    ].flatMap((extension) =>
-      getAllFiles(
-        searchOptions.deep,
-        src,
-        extension,
-        undefined,
-        undefined,
-        undefined,
-        src,
-        searchOptions.exclude,
-        searchOptions.noIgnore,
-      ),
-    );
+    srcFiles = getAllSrcJSAndTSFiles(src, deep);
   } catch {
     return { components: [], dependencies: [], services: [] };
   }