@cyclonedx/cdxgen 10.0.2 → 10.0.4

package/README.md

@@ -175,8 +175,9 @@ Options:
                                es.                    [boolean] [default: false]
       --spec-version           CycloneDX Specification version to use. Defaults
                                to 1.5                             [default: 1.5]
-      --filter                 Filter components containing this word in purl.
-                                Multiple values allowed.                 [array]
+      --filter                 Filter components containing this word in purl or
+                                component.properties.value. Multiple values allo
+                               wed.                                      [array]
       --only                   Include components only containing this word in
                                 purl. Useful to generate BOM with first party co
                                mponents alone. Multiple values allowed.  [array]

package/analyzer.js

@@ -8,7 +8,6 @@ import { basename, resolve, isAbsolute, relative } from "node:path";
 const IGNORE_DIRS = process.env.ASTGEN_IGNORE_DIRS
   ? process.env.ASTGEN_IGNORE_DIRS.split(",")
   : [
-      "node_modules",
       "venv",
       "docs",
       "test",
@@ -37,7 +36,7 @@ const IGNORE_FILE_PATTERN = new RegExp(
   "i"
 );
 
-const getAllFiles = (dir, extn, files, result, regex) => {
+const getAllFiles = (deep, dir, extn, files, result, regex) => {
   files = files || readdirSync(dir);
   result = result || [];
   regex = regex || new RegExp(`\\${extn}$`);
@@ -60,8 +59,20 @@ const getAllFiles = (dir, extn, files, result, regex) => {
       ) {
         continue;
       }
+      // We need to include node_modules in deep mode to track exports
+      // Ignore only for non-deep analysis
+      if (!deep && dirName === "node_modules") {
+        continue;
+      }
       try {
-        result = getAllFiles(file, extn, readdirSync(file), result, regex);
+        result = getAllFiles(
+          deep,
+          file,
+          extn,
+          readdirSync(file),
+          result,
+          regex
+        );
       } catch (error) {
         continue;
       }
@@ -101,7 +112,14 @@ const babelParserOptions = {
  * Filter only references to (t|jsx?) or (less|scss) files for now.
  * Opt to use our relative paths.
  */
-const setFileRef = (allImports, src, file, pathnode, specifiers = []) => {
+const setFileRef = (
+  allImports,
+  allExports,
+  src,
+  file,
+  pathnode,
+  specifiers = []
+) => {
   const pathway = pathnode.value || pathnode.name;
   const sourceLoc = pathnode.loc?.start;
   if (!pathway) {
@@ -115,9 +133,13 @@ const setFileRef = (allImports, src, file, pathnode, specifiers = []) => {
   const importedModules = specifiers
     .map((s) => s.imported?.name)
     .filter((v) => v !== undefined);
+  const exportedModules = specifiers
+    .map((s) => s.exported?.name)
+    .filter((v) => v !== undefined);
   const occurrence = {
     importedAs: pathway,
     importedModules,
+    exportedModules,
     isExternal: true,
     fileName: fileRelativeLoc,
     lineNumber: sourceLoc && sourceLoc.line ? sourceLoc.line : undefined,
@@ -132,10 +154,13 @@ const setFileRef = (allImports, src, file, pathnode, specifiers = []) => {
       moduleFullPath = relative(src, moduleFullPath);
       wasAbsolute = true;
     }
-    occurrence.isExternal = false;
+    if (!moduleFullPath.startsWith("node_modules/")) {
+      occurrence.isExternal = false;
+    }
   }
   allImports[moduleFullPath] = allImports[moduleFullPath] || new Set();
   allImports[moduleFullPath].add(occurrence);
+
   // Handle module package name
   // Eg: zone.js/dist/zone will be referred to as zone.js in package.json
   if (!wasAbsolute && moduleFullPath.includes("/")) {
@@ -143,6 +168,16 @@ const setFileRef = (allImports, src, file, pathnode, specifiers = []) => {
     allImports[modPkg] = allImports[modPkg] || new Set();
     allImports[modPkg].add(occurrence);
   }
+  if (exportedModules && exportedModules.length) {
+    moduleFullPath = moduleFullPath
+      .replace("node_modules/", "")
+      .replace("dist/", "")
+      .replace(/\.(js|ts|cjs|mjs)$/g, "")
+      .replace("src/", "");
+    allExports[moduleFullPath] = allExports[moduleFullPath] || new Set();
+    occurrence.exportedModules = exportedModules;
+    allExports[moduleFullPath].add(occurrence);
+  }
 };
 
 const vueCleaningRegex = /<\/*script.*>|<style[\s\S]*style>|<\/*br>/gi;
@@ -178,13 +213,14 @@ const fileToParseableCode = (file) => {
  * Check AST tree for any (j|tsx?) files and set a file
  * references for any import, require or dynamic import files.
  */
-const parseFileASTTree = (src, file, allImports) => {
+const parseFileASTTree = (src, file, allImports, allExports) => {
   const ast = parse(fileToParseableCode(file), babelParserOptions);
   traverse.default(ast, {
     ImportDeclaration: (path) => {
       if (path && path.node) {
         setFileRef(
           allImports,
+          allExports,
           src,
           file,
           path.node.source,
@@ -200,24 +236,25 @@ const parseFileASTTree = (src, file, allImports) => {
         path.node.name === "require" &&
         path.parent.type === "CallExpression"
       ) {
-        setFileRef(allImports, src, file, path.parent.arguments[0]);
+        setFileRef(allImports, allExports, src, file, path.parent.arguments[0]);
       }
     },
     // Use for dynamic imports like routes.jsx
     CallExpression: (path) => {
       if (path && path.node && path.node.callee.type === "Import") {
-        setFileRef(allImports, src, file, path.node.arguments[0]);
+        setFileRef(allImports, allExports, src, file, path.node.arguments[0]);
       }
     },
     // Use for export barrells
     ExportAllDeclaration: (path) => {
-      setFileRef(allImports, src, file, path.node.source);
+      setFileRef(allImports, allExports, src, file, path.node.source);
     },
     ExportNamedDeclaration: (path) => {
       // ensure there is a path export
       if (path && path.node && path.node.source) {
         setFileRef(
           allImports,
+          allExports,
           src,
           file,
           path.node.source,
@@ -231,36 +268,37 @@ const parseFileASTTree = (src, file, allImports) => {
 /**
  * Return paths to all (j|tsx?) files.
  */
-const getAllSrcJSAndTSFiles = (src) =>
+const getAllSrcJSAndTSFiles = (src, deep) =>
   Promise.all([
-    getAllFiles(src, ".js"),
-    getAllFiles(src, ".jsx"),
-    getAllFiles(src, ".cjs"),
-    getAllFiles(src, ".mjs"),
-    getAllFiles(src, ".ts"),
-    getAllFiles(src, ".tsx"),
-    getAllFiles(src, ".vue"),
-    getAllFiles(src, ".svelte")
+    getAllFiles(deep, src, ".js"),
+    getAllFiles(deep, src, ".jsx"),
+    getAllFiles(deep, src, ".cjs"),
+    getAllFiles(deep, src, ".mjs"),
+    getAllFiles(deep, src, ".ts"),
+    getAllFiles(deep, src, ".tsx"),
+    getAllFiles(deep, src, ".vue"),
+    getAllFiles(deep, src, ".svelte")
   ]);
 
 /**
- * Where Node CLI runs from.
+ * Find all imports and exports
  */
-export const findJSImports = async (src) => {
+export const findJSImportsExports = async (src, deep) => {
   const allImports = {};
+  const allExports = {};
   const errFiles = [];
   try {
-    const promiseMap = await getAllSrcJSAndTSFiles(src);
+    const promiseMap = await getAllSrcJSAndTSFiles(src, deep);
     const srcFiles = promiseMap.flatMap((d) => d);
     for (const file of srcFiles) {
       try {
-        parseFileASTTree(src, file, allImports);
+        parseFileASTTree(src, file, allImports, allExports);
       } catch (err) {
         errFiles.push(file);
       }
     }
-    return allImports;
+    return { allImports, allExports };
   } catch (err) {
-    return allImports;
+    return { allImports, allExports };
   }
 };

package/bin/cdxgen.js

@@ -193,7 +193,7 @@ const args = yargs(hideBin(process.argv))
   })
   .option("filter", {
     description:
-      "Filter components containing this word in purl. Multiple values allowed."
+      "Filter components containing this word in purl or component.properties.value. Multiple values allowed."
   })
   .option("only", {
     description:

package/index.js

@@ -133,7 +133,7 @@ const selfPJson = JSON.parse(
   readFileSync(join(dirName, "package.json"), "utf-8")
 );
 const _version = selfPJson.version;
-import { findJSImports } from "./analyzer.js";
+import { findJSImportsExports } from "./analyzer.js";
 import { gte, lte } from "semver";
 import {
   getPkgPathList,
@@ -1795,6 +1795,7 @@ export const createNodejsBom = async (path, options) => {
     }
   }
   let allImports = {};
+  let allExports = {};
   if (
     !["docker", "oci", "container", "os"].includes(options.projectType) &&
     !options.noBabel
@@ -1804,7 +1805,9 @@ export const createNodejsBom = async (path, options) => {
         `Performing babel-based package usage analysis with source code at ${path}`
       );
     }
-    allImports = await findJSImports(path);
+    const retData = await findJSImportsExports(path, options.deep);
+    allImports = retData.allImports;
+    allExports = retData.allExports;
   }
   const yarnLockFiles = getAllFiles(
     path,
@@ -1976,7 +1979,12 @@ export const createNodejsBom = async (path, options) => {
     if (existsSync(swFile)) {
       let pkgList = await parseNodeShrinkwrap(swFile);
       if (allImports && Object.keys(allImports).length) {
-        pkgList = addEvidenceForImports(pkgList, allImports);
+        pkgList = await addEvidenceForImports(
+          pkgList,
+          allImports,
+          allExports,
+          options.deep
+        );
       }
       return buildBomNSData(options, pkgList, "npm", {
         allImports,
@@ -1986,10 +1994,16 @@ export const createNodejsBom = async (path, options) => {
     } else if (existsSync(pnpmLock)) {
       let pkgList = await parsePnpmLock(pnpmLock);
       if (allImports && Object.keys(allImports).length) {
-        pkgList = addEvidenceForImports(pkgList, allImports);
+        pkgList = await addEvidenceForImports(
+          pkgList,
+          allImports,
+          allExports,
+          options.deep
+        );
       }
       return buildBomNSData(options, pkgList, "npm", {
         allImports,
+        allExports,
         src: path,
         filename: "pnpm-lock.yaml"
       });
@@ -2140,7 +2154,12 @@ export const createNodejsBom = async (path, options) => {
   // We need to set this to force our version to be used rather than the directory name based one.
   options.parentComponent = parentComponent;
   if (allImports && Object.keys(allImports).length) {
-    pkgList = addEvidenceForImports(pkgList, allImports);
+    pkgList = await addEvidenceForImports(
+      pkgList,
+      allImports,
+      allExports,
+      options.deep
+    );
   }
   return buildBomNSData(options, pkgList, "npm", {
     src: path,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cyclonedx/cdxgen",
-  "version": "10.0.2",
+  "version": "10.0.4",
   "description": "Creates CycloneDX Software Bill of Materials (SBOM) from source or container image",
   "homepage": "http://github.com/cyclonedx/cdxgen",
   "author": "Prabhu Subramanian <prabhu@appthreat.com>",
@@ -65,7 +65,7 @@
     "find-up": "7.0.0",
     "glob": "^10.3.10",
     "global-agent": "^3.0.0",
-    "got": "14.0.0",
+    "got": "14.1.0",
     "iconv-lite": "^0.6.3",
     "js-yaml": "^4.1.0",
     "jws": "^4.0.0",
@@ -82,7 +82,7 @@
     "yargs": "^17.7.2"
   },
   "optionalDependencies": {
-    "@appthreat/atom": "2.0.6",
+    "@appthreat/atom": "2.0.7",
     "@appthreat/cdx-proto": "^0.0.4",
     "@cyclonedx/cdxgen-plugins-bin": "^1.5.8",
     "@cyclonedx/cdxgen-plugins-bin-windows-amd64": "^1.5.8",

package/postgen.js

@@ -27,7 +27,10 @@ export const filterBom = (bomJson, options) => {
       }
       let purlfiltered = false;
       for (const filterstr of options.only) {
-        if (filterstr.length && !comp.purl.toLowerCase().includes(filterstr)) {
+        if (
+          filterstr.length &&
+          !comp.purl.toLowerCase().includes(filterstr.toLowerCase())
+        ) {
           filtered = true;
           purlfiltered = true;
           continue;
@@ -42,11 +45,29 @@ export const filterBom = (bomJson, options) => {
       }
       let purlfiltered = false;
       for (const filterstr of options.filter) {
-        if (filterstr.length && comp.purl.toLowerCase().includes(filterstr)) {
+        // Check the purl
+        if (
+          filterstr.length &&
+          comp.purl.toLowerCase().includes(filterstr.toLowerCase())
+        ) {
           filtered = true;
           purlfiltered = true;
           continue;
         }
+        // Look for any properties value matching the string
+        const properties = comp.properties || [];
+        for (const aprop of properties) {
+          if (
+            filterstr.length &&
+            aprop &&
+            aprop.value &&
+            aprop.value.toLowerCase().includes(filterstr.toLowerCase())
+          ) {
+            filtered = true;
+            purlfiltered = true;
+            continue;
+          }
+        }
       }
       if (!purlfiltered) {
         newPkgMap[comp["bom-ref"]] = comp;

package/postgen.test.js

@@ -41,14 +41,14 @@ test("filter bom tests2", () => {
       throw new Error(`${comp.purl} is unexpected`);
     }
   }
-  expect(newBom.components.length).toEqual(177);
+  expect(newBom.components.length).toEqual(158);
   newBom = filterBom(bomJson, { filter: ["apache", "json"] });
   for (const comp of newBom.components) {
     if (comp.purl.includes("apache") || comp.purl.includes("json")) {
       throw new Error(`${comp.purl} is unexpected`);
     }
   }
-  expect(newBom.components.length).toEqual(172);
+  expect(newBom.components.length).toEqual(135);
   expect(newBom.compositions).toBeUndefined();
   newBom = filterBom(bomJson, {
     only: ["org.springframework"],
@@ -60,7 +60,7 @@ test("filter bom tests2", () => {
       throw new Error(`${comp.purl} is unexpected`);
     }
   }
-  expect(newBom.components.length).toEqual(37);
+  expect(newBom.components.length).toEqual(29);
   expect(newBom.compositions).toEqual([
     {
       aggregate: "incomplete_first_party_only",

package/utils.js

@@ -559,13 +559,29 @@ export const parsePkgJson = async (pkgJsonFile, simple = false) => {
         null,
         null
       ).toString();
+      const author = pkgData.author;
+      const authorString =
+        author instanceof Object
+          ? `${author.name}${author.email ? ` <${author.email}>` : ""}${
+              author.url ? ` (${author.url})` : ""
+            }`
+          : author;
       const apkg = {
         name,
         group,
         version: pkgData.version,
+        description: pkgData.description,
         purl: purl,
-        "bom-ref": decodeURIComponent(purl)
+        "bom-ref": decodeURIComponent(purl),
+        author: authorString,
+        license: pkgData.license
       };
+      if (pkgData.homepage) {
+        apkg.homepage = { url: pkgData.homepage };
+      }
+      if (pkgData.repository && pkgData.repository.url) {
+        apkg.repository = { url: pkgData.repository.url };
+      }
       if (!simple) {
         apkg.properties = [
           {
@@ -720,6 +736,12 @@ export const parsePkgLock = async (pkgLockFile, options = {}) => {
           value: node.resolved
         });
       }
+      if (node.location) {
+        pkg.properties.push({
+          name: "LocalNodeModulesPath",
+          value: node.location
+        });
+      }
     }
     const packageLicense = node.package.license;
     if (packageLicense) {
@@ -1185,8 +1207,11 @@ export const parseYarnLock = async function (yarnLockFile) {
         }
         // checksum used by yarn 2/3 is hex encoded
         if (l.startsWith("checksum")) {
+          // in some cases yarn 4 will add a prefix to the checksum, containing the cachekey and compression level
+          // example: 10c0/53c2b231a61a46792b39a0d43bc4f4f77...
+          const checksum = parts[1].split("/").pop();
           integrity =
-            "sha512-" + Buffer.from(parts[1], "hex").toString("base64");
+            "sha512-" + Buffer.from(checksum, "hex").toString("base64");
         }
         if (l.startsWith("resolved")) {
           const tmpB = parts[1].split("#");
@@ -8132,9 +8157,16 @@ export const parsePackageJsonName = (name) => {
  *
  * @param {array} pkgList List of package
  * @param {object} allImports Import statements object with package name as key and an object with file and location details
+ * @param {object} allExports Exported modules if available from node_modules
  */
-export const addEvidenceForImports = (pkgList, allImports) => {
+export const addEvidenceForImports = async (
+  pkgList,
+  allImports,
+  allExports,
+  deep
+) => {
   const impPkgs = Object.keys(allImports);
+  const exportedPkgs = Object.keys(allExports);
   for (const pkg of pkgList) {
     if (impPkgs && impPkgs.length) {
       // Assume that all packages are optional until we see an evidence
@@ -8155,6 +8187,47 @@ export const addEvidenceForImports = (pkgList, allImports) => {
           find_pkg.startsWith(alias) &&
           (find_pkg.length === alias.length || find_pkg[alias.length] === "/")
       );
+      const all_exports = exportedPkgs.filter((find_pkg) =>
+        find_pkg.startsWith(alias)
+      );
+      if (all_exports && all_exports.length) {
+        let exportedModules = new Set(all_exports);
+        pkg.properties = pkg.properties || [];
+        for (const subevidence of all_exports) {
+          const evidences = allExports[subevidence];
+          for (const evidence of evidences) {
+            if (evidence && Object.keys(evidence).length) {
+              if (evidence.exportedModules.length > 1) {
+                for (const aexpsubm of evidence.exportedModules) {
+                  // Be selective on the submodule names
+                  if (
+                    !evidence.importedAs
+                      .toLowerCase()
+                      .includes(aexpsubm.toLowerCase()) &&
+                    !alias.endsWith(aexpsubm)
+                  ) {
+                    // Store both the short and long form of the exported sub modules
+                    if (aexpsubm.length > 3) {
+                      exportedModules.add(aexpsubm);
+                    }
+                    exportedModules.add(
+                      `${evidence.importedAs.replace("./", "")}/${aexpsubm}`
+                    );
+                  }
+                }
+              }
+            }
+          }
+        }
+        exportedModules = Array.from(exportedModules);
+        if (exportedModules.length) {
+          pkg.properties.push({
+            name: "ExportedModules",
+            value: exportedModules.join(",")
+          });
+        }
+      }
+      // Identify all the imported modules of a component
       if (impPkgs.includes(alias) || all_includes.length) {
         let importedModules = new Set();
         pkg.scope = "required";
@@ -8175,7 +8248,9 @@ export const addEvidenceForImports = (pkgList, allImports) => {
                   continue;
                 }
                 // Store both the short and long form of the imported sub modules
-                importedModules.add(importedSm);
+                if (importedSm.length > 3) {
+                  importedModules.add(importedSm);
+                }
                 importedModules.add(`${evidence.importedAs}/${importedSm}`);
               }
             }
@@ -8191,8 +8266,35 @@ export const addEvidenceForImports = (pkgList, allImports) => {
         }
         break;
       }
-    }
-  }
+      // Capture metadata such as description from local node_modules in deep mode
+      if (deep && !pkg.description && pkg.properties) {
+        let localNodeModulesPath = undefined;
+        for (const aprop of pkg.properties) {
+          if (aprop.name === "LocalNodeModulesPath") {
+            localNodeModulesPath = resolve(join(aprop.value, "package.json"));
+            break;
+          }
+        }
+        if (localNodeModulesPath && existsSync(localNodeModulesPath)) {
+          const lnmPkgList = await parsePkgJson(localNodeModulesPath, true);
+          if (lnmPkgList && lnmPkgList.length === 1) {
+            const lnmMetadata = lnmPkgList[0];
+            if (lnmMetadata && Object.keys(lnmMetadata).length) {
+              pkg.description = lnmMetadata.description;
+              pkg.author = lnmMetadata.author;
+              pkg.license = lnmMetadata.license;
+              pkg.homepage = lnmMetadata.homepage;
+              pkg.repository = lnmMetadata.repository;
+            }
+          }
+        }
+      }
+    } // for alias
+    // Trim the properties
+    pkg.properties = pkg.properties.filter(
+      (p) => p.name !== "LocalNodeModulesPath"
+    );
+  } // for pkg
   return pkgList;
 };
 

package/utils.test.js

@@ -38,6 +38,7 @@ import {
   parsePom,
   getMvnMetadata,
   getLicenses,
+  parsePkgJson,
   parsePkgLock,
   parseBowerJson,
   parseNodeShrinkwrap,
@@ -1719,6 +1720,11 @@ test("get licenses", () => {
   ]);
 });
 
+test("parsePkgJson", async () => {
+  const pkgList = await parsePkgJson("./package.json", true);
+  expect(pkgList.length).toEqual(1);
+});
+
 test("parsePkgLock v1", async () => {
   const parsedList = await parsePkgLock(
     "./test/data/package-json/v1/package-lock.json"
@@ -2399,6 +2405,18 @@ test("parseYarnLock", async () => {
   expect(parsedList.pkgList[0]["bom-ref"]).toEqual(
     "pkg:npm/@aashutoshrathi/word-wrap@1.2.6"
   );
+  parsedList = await parseYarnLock("./test/data/yarn_locks/yarnv4.1.lock");
+  expect(parsedList.pkgList.length).toEqual(861);
+  expect(parsedList.dependenciesList.length).toEqual(858);
+  expect(parsedList.pkgList[0].purl).toEqual(
+    "pkg:npm/%40aashutoshrathi/word-wrap@1.2.6"
+  );
+  expect(parsedList.pkgList[0]["bom-ref"]).toEqual(
+    "pkg:npm/@aashutoshrathi/word-wrap@1.2.6"
+  );
+  expect(parsedList.pkgList[0]._integrity).toEqual(
+    "sha512-U8KyMaYaRnkrOaDUO8T093a7RUKqV+4EkwZ2gC5VASgsL8iqwU5M0fESD/i1Jha2/1q1Oa0wqiJ31yZES3Fhnw=="
+  );
 });
 
 test("parseComposerLock", () => {