@cyclonedx/cdxgen 9.6.0 → 9.7.0

package/index.js

@@ -120,12 +120,13 @@ import {
   executeOsQuery,
   getOSPackages
 } from "./binary.js";
-const osQueries = JSON.parse(
-  readFileSync(join(dirName, "data", "queries.json"))
-);
 
 const isWin = _platform() === "win32";
 
+const osQueries = !isWin
+  ? JSON.parse(readFileSync(join(dirName, "data", "queries.json")))
+  : JSON.parse(readFileSync(join(dirName, "data", "queries-win.json")));
+
 import { table } from "table";
 
 // Construct gradle cache directory
@@ -372,7 +373,16 @@ function addMetadata(parentComponent = {}, format = "xml", options = {}) {
           // We cannot use purl or bom-ref here since they would not match
           // purl - could have application on one side and a different type
           // bom-ref could have qualifiers on one side
-          if (fullName !== parentFullName) {
+          // Ignore components that have the same name as the parent component but with latest as the version.
+          // These are default components created based on directory names
+          if (
+            fullName !== parentFullName &&
+            !(
+              (comp.name === parentComponent.name ||
+                comp.name === parentComponent.name + ":latest") &&
+              comp.version === "latest"
+            )
+          ) {
             if (!comp["bom-ref"]) {
               comp["bom-ref"] = `pkg:${comp.type}/${fullName}`;
             }
@@ -380,6 +390,7 @@ function addMetadata(parentComponent = {}, format = "xml", options = {}) {
           }
         }
       } // for
+      parentComponent.components = subComponents;
     }
     if (format === "json") {
       metadata.component = parentComponent;
@@ -650,7 +661,6 @@ function addComponent(
       return;
     }
     const licenses = pkg.licenses || getLicenses(pkg, format);
-
     const purl =
       pkg.purl ||
       new PackageURL(
@@ -745,8 +755,22 @@ function addComponent(
  * word for it, otherwise, identify the module as a 'library'.
  */
 function determinePackageType(pkg) {
-  if (pkg.type === "application") {
-    return "application";
+  // Retain the exact component type in certain cases.
+  if (
+    [
+      "application",
+      "container",
+      "platform",
+      "operating-system",
+      "device",
+      "device-driver",
+      "firmware",
+      "file",
+      "machine-learning-model",
+      "data"
+    ].includes(pkg.type)
+  ) {
+    return pkg.type;
   }
   if (pkg.purl) {
     try {
@@ -1174,7 +1198,8 @@ export const createJavaBom = async (path, options) => {
           cwd: basePath,
           shell: true,
           encoding: "utf-8",
-          timeout: TIMEOUT_MS
+          timeout: TIMEOUT_MS,
+          maxBuffer: 50 * 1024 * 1024
         });
         // Check if the cyclonedx plugin created the required bom.xml file
         // Sometimes the plugin fails silently for complex maven projects
@@ -2199,10 +2224,13 @@ export const createPythonBom = async (path, options) => {
   if (pdmLockFiles && pdmLockFiles.length) {
     poetryFiles = poetryFiles.concat(pdmLockFiles);
   }
-  const reqFiles = getAllFiles(
+  let reqFiles = getAllFiles(
     path,
     (options.multiProject ? "**/" : "") + "*requirements*.txt"
   );
+  reqFiles = reqFiles.filter(
+    (f) => !f.includes(join("mercurial", "helptext", "internals"))
+  );
   const reqDirFiles = getAllFiles(
     path,
     (options.multiProject ? "**/" : "") + "requirements/*.txt"
@@ -3131,34 +3159,44 @@ export const createCloudBuildBom = (path, options) => {
 };
 
 /**
- * Function to create bom string for the current OS using osquery
+ * Function to create obom string for the current OS using osquery
  *
  * @param path to the project
  * @param options Parse options from the cli
  */
 export const createOSBom = (path, options) => {
   console.warn(
-    "About to generate SBoM for the current OS installation. This would take several minutes ..."
+    "About to generate OBoM for the current OS installation. This will take several minutes ..."
   );
   let pkgList = [];
   let bomData = {};
+  let parentComponent = {};
   for (const queryCategory of Object.keys(osQueries)) {
     const queryObj = osQueries[queryCategory];
     const results = executeOsQuery(queryObj.query);
     const dlist = convertOSQueryResults(queryCategory, queryObj, results);
     if (dlist && dlist.length) {
-      pkgList = pkgList.concat(dlist);
+      if (!Object.keys(parentComponent).length) {
+        parentComponent = dlist.splice(0, 1)[0];
+      }
+      pkgList = pkgList.concat(
+        dlist.sort(function (a, b) {
+          return a.name.localeCompare(b.name);
+        })
+      );
     }
   } // for
   if (pkgList.length) {
     bomData = buildBomNSData(options, pkgList, "", {
       src: "",
-      filename: ""
+      filename: "",
+      parentComponent
     });
   }
   options.bomData = bomData;
   options.multiProject = true;
   options.installDeps = false;
+  options.parentComponent = parentComponent;
   // Force the project type to os
   options.projectType = "os";
   options.lastWorkingDir = undefined;
@@ -3910,7 +3948,7 @@ export const mergeDependencies = (
   for (const akey of Object.keys(deps_map)) {
     retlist.push({
       ref: akey,
-      dependsOn: Array.from(deps_map[akey])
+      dependsOn: Array.from(deps_map[akey]).sort()
     });
   }
   return retlist;
@@ -4003,7 +4041,7 @@ export const createMultiXBom = async (pathList, options) => {
     ["docker", "oci", "container"].includes(options.projectType) &&
     options.allLayersExplodedDir
   ) {
-    const { osPackages, allTypes } = getOSPackages(
+    const { osPackages, dependenciesList, allTypes } = getOSPackages(
       options.allLayersExplodedDir
     );
     if (DEBUG_MODE) {
@@ -4018,6 +4056,17 @@ export const createMultiXBom = async (pathList, options) => {
     componentsXmls = componentsXmls.concat(
       listComponents(options, {}, osPackages, "", "xml")
     );
+    if (dependenciesList && dependenciesList.length) {
+      dependencies = dependencies.concat(dependenciesList);
+    }
+    if (parentComponent && Object.keys(parentComponent).length) {
+      // Make the parent oci image depend on all os components
+      const parentDependsOn = new Set(osPackages.map((p) => p["bom-ref"]));
+      dependencies.splice(0, 0, {
+        ref: parentComponent["bom-ref"],
+        dependsOn: Array.from(parentDependsOn).sort()
+      });
+    }
   }
   if (options.projectType === "os" && options.bomData) {
     bomData = options.bomData;
@@ -4409,7 +4458,12 @@ export const createMultiXBom = async (pathList, options) => {
     // Jar scanning is enabled by default
     // See #330
     bomData = createJarBom(path, options);
-    if (bomData && bomData.bomJson && bomData.bomJson.components) {
+    if (
+      bomData &&
+      bomData.bomJson &&
+      bomData.bomJson.components &&
+      bomData.bomJson.components.length
+    ) {
       if (DEBUG_MODE) {
         console.log(
           `Found ${bomData.bomJson.components.length} jar packages at ${path}`
@@ -4430,7 +4484,12 @@ export const createMultiXBom = async (pathList, options) => {
   } // for
   if (options.lastWorkingDir && options.lastWorkingDir !== "") {
     bomData = createJarBom(options.lastWorkingDir, options);
-    if (bomData && bomData.bomJson && bomData.bomJson.components) {
+    if (
+      bomData &&
+      bomData.bomJson &&
+      bomData.bomJson.components &&
+      bomData.bomJson.components.length
+    ) {
       if (DEBUG_MODE) {
         console.log(
           `Found ${bomData.bomJson.components.length} jar packages at ${options.lastWorkingDir}`
@@ -4462,7 +4521,8 @@ export const createMultiXBom = async (pathList, options) => {
     parentComponent.components = trimComponents(parentSubComponents, "json");
     if (
       parentComponent.components.length == 1 &&
-      parentComponent.components[0].name == parentComponent.name
+      parentComponent.components[0].name == parentComponent.name &&
+      !parentComponent.purl.startsWith("pkg:container")
     ) {
       parentComponent = parentComponent.components[0];
       delete parentComponent.components;
@@ -4818,8 +4878,26 @@ export const createBom = async (path, options) => {
             purl: "pkg:oci/" + inspectData.RepoDigests[0],
             _integrity: inspectData.RepoDigests[0].replace("sha256:", "sha256-")
           };
+          options.parentComponent["bom-ref"] = options.parentComponent.purl;
         }
+      } else if (inspectData.Id) {
+        options.parentComponent = {
+          name: inspectData.RepoDigests[0].split("@")[0],
+          version: inspectData.RepoDigests[0]
+            .split("@")[1]
+            .replace("sha256:", ""),
+          type: "container",
+          purl: "pkg:oci/" + inspectData.RepoDigests[0],
+          _integrity: inspectData.RepoDigests[0].replace("sha256:", "sha256-")
+        };
+        options.parentComponent["bom-ref"] = options.parentComponent.purl;
       }
+    } else {
+      options.parentComponent = createDefaultParentComponent(
+        path,
+        "container",
+        options
+      );
     }
     // Pass the entire export data about the image layers
     options.exportData = exportData;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cyclonedx/cdxgen",
-  "version": "9.6.0",
+  "version": "9.7.0",
   "description": "Creates CycloneDX Software Bill-of-Materials (SBOM) from source or container image",
   "homepage": "http://github.com/cyclonedx/cdxgen",
   "author": "Prabhu Subramanian <prabhu@appthreat.com>",
@@ -53,8 +53,8 @@
     "url": "https://github.com/cyclonedx/cdxgen/issues"
   },
   "dependencies": {
-    "@babel/parser": "^7.22.10",
-    "@babel/traverse": "^7.22.10",
+    "@babel/parser": "^7.22.11",
+    "@babel/traverse": "^7.22.11",
     "ajv": "^8.12.0",
     "ajv-formats": "^2.1.1",
     "cheerio": "^1.0.0-rc.12",
@@ -68,7 +68,7 @@
     "node-stream-zip": "^1.15.0",
     "packageurl-js": "^1.0.2",
     "prettify-xml": "^1.2.0",
-    "properties-reader": "^2.2.0",
+    "properties-reader": "^2.3.0",
     "semver": "^7.5.3",
     "ssri": "^10.0.4",
     "table": "^6.8.1",
@@ -80,7 +80,9 @@
   },
   "optionalDependencies": {
     "@appthreat/atom": "^1.1.4",
-    "@cyclonedx/cdxgen-plugins-bin": "^1.2.0",
+    "@cyclonedx/cdxgen-plugins-bin": "^1.4.0",
+    "@cyclonedx/cdxgen-plugins-bin-arm64": "^1.4.0",
+    "@cyclonedx/cdxgen-plugins-bin-ppc64": "^1.4.0",
     "body-parser": "^1.20.2",
     "compression": "^1.7.4",
     "connect": "^3.7.0",
@@ -95,8 +97,8 @@
   ],
   "devDependencies": {
     "caxa": "^3.0.1",
-    "eslint": "^8.47.0",
-    "jest": "^29.5.0",
-    "prettier": "3.0.1"
+    "eslint": "^8.48.0",
+    "jest": "^29.6.4",
+    "prettier": "3.0.2"
   }
 }

package/utils.js

@@ -329,21 +329,21 @@ export const getNpmMetadata = async function (pkgList) {
 const _getDepPkgList = async function (
   pkgLockFile,
   pkgList,
-  dependenciesList,
   depKeys,
-  pkg
+  pkg,
+  versionCache
 ) {
-  const pkgDependencies =
-    pkg.lockfileVersion && pkg.lockfileVersion >= 3
-      ? pkg.packages
-      : pkg.dependencies;
-  const versionCache = {};
+  const pkgDependencies = {
+    ...(pkg.packages || {}),
+    ...(pkg.dependencies || {})
+  };
   if (pkg.packages) {
     for (const k in pkg.packages) {
       if (k === "") {
         continue;
       }
       const pl = pkg.packages[k];
+      versionCache[k] = pl.version;
       versionCache[k.replaceAll("node_modules/", "")] = pl.version;
     }
   }
@@ -355,7 +355,13 @@ const _getDepPkgList = async function (
         continue;
       }
       const name = k;
-      const version = pkgDependencies[name].version;
+      let version = pkgDependencies[name].version;
+      if (!version && versionCache[k]) {
+        version = versionCache[k];
+      }
+      if (!version && pkgDependencies["node_modules/" + name]) {
+        version = pkgDependencies["node_modules/" + name].version;
+      }
       const purl = new PackageURL(
         "npm",
         "",
@@ -415,32 +421,20 @@ const _getDepPkgList = async function (
           const deppurlString = decodeURIComponent(deppurl.toString());
           deplist.push(deppurlString);
         }
-        if (!depKeys[purlString]) {
-          dependenciesList.push({
-            ref: purlString,
-            dependsOn: deplist
-          });
-          depKeys[purlString] = true;
-        }
+        depKeys[purlString] = (depKeys[purlString] || []).concat(deplist);
         if (pkg.lockfileVersion && pkg.lockfileVersion >= 3) {
           // Do not recurse for lock file v3 and above
         } else {
           await _getDepPkgList(
             pkgLockFile,
             pkgList,
-            dependenciesList,
             depKeys,
-            pkgDependencies[name]
+            pkgDependencies[name],
+            versionCache
           );
         }
-      } else {
-        if (!depKeys[purlString]) {
-          dependenciesList.push({
-            ref: purlString,
-            dependsOn: []
-          });
-          depKeys[purlString] = true;
-        }
+      } else if (!depKeys[purlString]) {
+        depKeys[purlString] = [];
       }
     }
   }
@@ -519,6 +513,7 @@ export const parsePkgLock = async (pkgLockFile, options = {}) => {
   const dependenciesList = [];
   const depKeys = {};
   let rootPkg = {};
+  const versionCache = {};
   if (!options) {
     options = {};
   }
@@ -612,11 +607,17 @@ export const parsePkgLock = async (pkgLockFile, options = {}) => {
     pkgList = await _getDepPkgList(
       pkgLockFile,
       pkgList,
-      dependenciesList,
       depKeys,
-      lockData
+      lockData,
+      versionCache
     );
   }
+  for (const dk of Object.keys(depKeys)) {
+    dependenciesList.push({
+      ref: dk,
+      dependsOn: depKeys[dk] || []
+    });
+  }
   if (fetchLicenses && pkgList && pkgList.length) {
     if (DEBUG_MODE) {
       console.log(
@@ -2007,7 +2008,8 @@ export const guessLicenseId = function (content) {
  * @param {Array} pkgList Package list
  */
 export const getMvnMetadata = async function (pkgList) {
-  const MAVEN_CENTRAL_URL = "https://repo1.maven.org/maven2/";
+  const MAVEN_CENTRAL_URL =
+    process.env.MAVEN_CENTRAL_URL || "https://repo1.maven.org/maven2/";
   const ANDROID_MAVEN = "https://maven.google.com/";
   const cdepList = [];
   if (!pkgList || !pkgList.length) {
@@ -3285,14 +3287,15 @@ export const getCratesMetadata = async function (pkgList) {
  * @param {Array} pkgList Package list
  */
 export const getDartMetadata = async function (pkgList) {
-  const PUB_DEV_URL = "https://pub.dev/api/packages/";
+  const PUB_DEV_URL = process.env.PUB_DEV_URL || "https://pub.dev";
+  const PUB_PACKAGES_URL = PUB_DEV_URL + "/api/packages/";
   const cdepList = [];
   for (const p of pkgList) {
     try {
       if (DEBUG_MODE) {
-        console.log(`Querying pub.dev for ${p.name}`);
+        console.log(`Querying ${PUB_DEV_URL} for ${p.name}`);
       }
-      const res = await cdxgenAgent.get(PUB_DEV_URL + p.name, {
+      const res = await cdxgenAgent.get(PUB_PACKAGES_URL + p.name, {
         responseType: "json",
         headers: {
           Accept: "application/vnd.pub.v2+json"
@@ -3310,7 +3313,7 @@ export const getDartMetadata = async function (pkgList) {
             if (pubspec.homepage) {
               p.homepage = { url: pubspec.homepage };
             }
-            p.license = "https://pub.dev/packages/" + p.name + "/license";
+            p.license = `${PUB_DEV_URL}/packages/${p.name}/license`;
             cdepList.push(p);
             break;
           }
@@ -4764,49 +4767,99 @@ export const convertOSQueryResults = function (
   const pkgList = [];
   if (results && results.length) {
     for (const res of results) {
-      if (res.version) {
-        const version = res.version;
-        let name = res.name || res.device_id;
-        const group = "";
-        const subpath = res.path || res.admindir || res.source;
-        const publisher = res.maintainer || res.creator;
-        let scope = undefined;
-        const compScope = res.priority;
-        if (["required", "optional", "excluded"].includes(compScope)) {
-          scope = compScope;
-        }
-        const description =
-          res.description ||
-          res.arguments ||
-          res.device ||
-          res.codename ||
-          res.section ||
-          res.status ||
-          res.identifier ||
-          res.components;
-        // Re-use the name from query obj
-        if (!name && results.length === 1 && queryObj.name) {
-          name = queryObj.name;
-        }
-        if (name && version) {
-          const purl = new PackageURL(
-            queryObj.purlType || "swid",
-            group,
-            name,
-            version,
-            undefined,
-            subpath
-          );
-          pkgList.push({
-            name,
-            group,
-            version,
-            description,
-            publisher,
-            purl,
-            scope
-          });
+      const version =
+        res.version ||
+        res.hotfix_id ||
+        res.hardware_version ||
+        res.port ||
+        res.pid ||
+        res.subject_key_id ||
+        res.interface ||
+        res.instance_id;
+      let name =
+        res.name ||
+        res.device_id ||
+        res.hotfix_id ||
+        res.uuid ||
+        res.serial ||
+        res.pid ||
+        res.address ||
+        res.ami_id ||
+        res.interface ||
+        res.client_app_id;
+      let group = "";
+      const subpath = res.path || res.admindir || res.source;
+      let publisher =
+        res.publisher ||
+        res.maintainer ||
+        res.creator ||
+        res.manufacturer ||
+        res.provider ||
+        "";
+      if (publisher === "null") {
+        publisher = "";
+      }
+      let scope = undefined;
+      const compScope = res.priority;
+      if (["required", "optional", "excluded"].includes(compScope)) {
+        scope = compScope;
+      }
+      const description =
+        res.description ||
+        res.summary ||
+        res.arguments ||
+        res.device ||
+        res.codename ||
+        res.section ||
+        res.status ||
+        res.identifier ||
+        res.components ||
+        "";
+      // Re-use the name from query obj
+      if (!name && results.length === 1 && queryObj.name) {
+        name = queryObj.name;
+      }
+      let qualifiers = undefined;
+      if (res.identifying_number && res.identifying_number.length) {
+        qualifiers = {
+          tag_id: res.identifying_number.replace("{", "").replace("}", "")
+        };
+      }
+      if (name) {
+        name = name.replace(/ /g, "+").replace(/[:%]/g, "-");
+        group = group.replace(/ /g, "+").replace(/[:%]/g, "-");
+        const purl = new PackageURL(
+          queryObj.purlType || "swid",
+          group,
+          name,
+          version || "",
+          qualifiers,
+          subpath
+        ).toString();
+        const apkg = {
+          name,
+          group,
+          version: version || "",
+          description,
+          publisher,
+          "bom-ref": decodeURIComponent(purl),
+          purl,
+          scope,
+          type: queryObj.componentType
+        };
+        const props = [{ name: "cdx:osquery:category", value: queryCategory }];
+        for (const k of Object.keys(res).filter(
+          (p) => !["name", "version", "description", "publisher"].includes(p)
+        )) {
+          if (res[k] && res[k] !== "null") {
+            props.push({
+              name: k,
+              value: res[k]
+            });
+          }
         }
+        apkg.properties = props;
+        pkgList.push(apkg);
       }
     }
   }
@@ -5783,27 +5836,37 @@ export const executeAtom = (src, args) => {
   }
   const freeMemoryGB = Math.floor(freemem() / 1024 / 1024 / 1024);
   if (DEBUG_MODE) {
-    console.log("Execuing", ATOM_BIN, args.join(" "));
+    console.log("Executing", ATOM_BIN, args.join(" "));
   }
   const env = {
     ...process.env,
     JAVA_OPTS: `-Xms${freeMemoryGB}G -Xmx${freeMemoryGB}G`
   };
+  env.PATH = `${env.PATH}${_delimiter}${join(
+    dirNameStr,
+    "node_modules",
+    ".bin"
+  )}`;
   const result = spawnSync(ATOM_BIN, args, {
     cwd,
     encoding: "utf-8",
     timeout: TIMEOUT_MS,
     env
   });
-  if (
-    result.stderr &&
-    result.stderr.includes(
-      "has been compiled by a more recent version of the Java Runtime"
-    )
-  ) {
-    console.log(
-      "Atom requires Java 17 or above. Please install a suitable version and re-run cdxgen to improve the SBoM accuracy.\nAlternatively, use the cdxgen container image."
-    );
+  if (result.stderr) {
+    if (
+      result.stderr.includes(
+        "has been compiled by a more recent version of the Java Runtime"
+      )
+    ) {
+      console.log(
+        "Atom requires Java 17 or above. Please install a suitable version and re-run cdxgen to improve the SBoM accuracy.\nAlternatively, use the cdxgen container image."
+      );
+    } else if (result.stderr.includes("astgen")) {
+      console.warn(
+        "WARN: Unable to locate astgen command. Install atom globally using sudo npm install -g @appthreat/atom to resolve this issue."
+      );
+    }
   }
   if (DEBUG_MODE) {
     if (result.stdout) {
@@ -5813,7 +5876,7 @@ export const executeAtom = (src, args) => {
       console.log(result.stderr);
     }
   }
-  return true;
+  return !result.error;
 };
 
 /**
@@ -6232,3 +6295,14 @@ export const addEvidenceForImports = (pkgList, allImports) => {
   }
   return pkgList;
 };
+
+export const componentSorter = (a, b) => {
+  if (a && b) {
+    for (const k of ["bom-ref", "purl", "name"]) {
+      if (a[k] && b[k]) {
+        return a[k].localeCompare(b[k]);
+      }
+    }
+  }
+  return a.localeCompare(b);
+};

package/utils.test.js

@@ -1358,8 +1358,8 @@ test("parsePkgLock", async () => {
   });
   parsedList = await parsePkgLock("./test/data/package-lock-v2.json");
   deps = parsedList.pkgList;
-  expect(deps.length).toEqual(1467);
-  expect(parsedList.dependenciesList.length).toEqual(1280);
+  expect(deps.length).toEqual(5433);
+  expect(parsedList.dependenciesList.length).toEqual(1616);
   expect(deps[0]).toEqual({
     "bom-ref": "pkg:npm/flink-dashboard@2.0.0",
     group: "",