@cyclonedx/cdxgen 9.6.1 → 9.7.1

package/utils.js

@@ -329,21 +329,21 @@ export const getNpmMetadata = async function (pkgList) {
 const _getDepPkgList = async function (
   pkgLockFile,
   pkgList,
-  dependenciesList,
   depKeys,
-  pkg
+  pkg,
+  versionCache
 ) {
-  const pkgDependencies =
-    pkg.lockfileVersion && pkg.lockfileVersion >= 3
-      ? pkg.packages
-      : pkg.dependencies;
-  const versionCache = {};
+  const pkgDependencies = {
+    ...(pkg.packages || {}),
+    ...(pkg.dependencies || {})
+  };
   if (pkg.packages) {
     for (const k in pkg.packages) {
       if (k === "") {
         continue;
       }
       const pl = pkg.packages[k];
+      versionCache[k] = pl.version;
       versionCache[k.replaceAll("node_modules/", "")] = pl.version;
     }
   }
@@ -355,7 +355,13 @@ const _getDepPkgList = async function (
         continue;
       }
       const name = k;
-      const version = pkgDependencies[name].version;
+      let version = pkgDependencies[name].version;
+      if (!version && versionCache[k]) {
+        version = versionCache[k];
+      }
+      if (!version && pkgDependencies["node_modules/" + name]) {
+        version = pkgDependencies["node_modules/" + name].version;
+      }
       const purl = new PackageURL(
         "npm",
         "",
@@ -415,32 +421,20 @@ const _getDepPkgList = async function (
           const deppurlString = decodeURIComponent(deppurl.toString());
           deplist.push(deppurlString);
         }
-        if (!depKeys[purlString]) {
-          dependenciesList.push({
-            ref: purlString,
-            dependsOn: deplist
-          });
-          depKeys[purlString] = true;
-        }
+        depKeys[purlString] = (depKeys[purlString] || []).concat(deplist);
         if (pkg.lockfileVersion && pkg.lockfileVersion >= 3) {
           // Do not recurse for lock file v3 and above
         } else {
           await _getDepPkgList(
             pkgLockFile,
             pkgList,
-            dependenciesList,
             depKeys,
-            pkgDependencies[name]
+            pkgDependencies[name],
+            versionCache
           );
         }
-      } else {
-        if (!depKeys[purlString]) {
-          dependenciesList.push({
-            ref: purlString,
-            dependsOn: []
-          });
-          depKeys[purlString] = true;
-        }
+      } else if (!depKeys[purlString]) {
+        depKeys[purlString] = [];
       }
     }
   }
@@ -519,6 +513,7 @@ export const parsePkgLock = async (pkgLockFile, options = {}) => {
   const dependenciesList = [];
   const depKeys = {};
   let rootPkg = {};
+  const versionCache = {};
   if (!options) {
     options = {};
   }
@@ -612,11 +607,17 @@ export const parsePkgLock = async (pkgLockFile, options = {}) => {
     pkgList = await _getDepPkgList(
       pkgLockFile,
       pkgList,
-      dependenciesList,
       depKeys,
-      lockData
+      lockData,
+      versionCache
     );
   }
+  for (const dk of Object.keys(depKeys)) {
+    dependenciesList.push({
+      ref: dk,
+      dependsOn: depKeys[dk] || []
+    });
+  }
   if (fetchLicenses && pkgList && pkgList.length) {
     if (DEBUG_MODE) {
       console.log(
@@ -1338,7 +1339,7 @@ export const parseMavenTree = function (rawOutput) {
             pkgArr[0],
             pkgArr[1],
             versionStr,
-            { type: "jar" },
+            { type: pkgArr[2] },
             null
           ).toString();
           purlString = decodeURIComponent(purlString);
@@ -1346,7 +1347,7 @@ export const parseMavenTree = function (rawOutput) {
             group: pkgArr[0],
             name: pkgArr[1],
             version: versionStr,
-            qualifiers: { type: "jar" }
+            qualifiers: { type: pkgArr[2] }
           });
           if (!level_trees[purlString]) {
             level_trees[purlString] = [];
@@ -1877,8 +1878,9 @@ export const parseBazelSkyframe = function (rawOutput) {
             // Remove the protocol, registry url and then file name
             let prefix_slice_count = 2;
             // Bug: #169
-            if (l.includes("/maven2/")) {
-              prefix_slice_count = 3;
+            const prefix = process.env.BAZEL_STRIP_MAVEN_PREFIX || "/maven2/";
+            if (l.includes(prefix)) {
+              prefix_slice_count = prefix.split("/").length;
             }
             jarPathParts = jarPathParts.slice(prefix_slice_count, -1);
             // The last part would be the version
@@ -2007,7 +2009,8 @@ export const guessLicenseId = function (content) {
  * @param {Array} pkgList Package list
  */
 export const getMvnMetadata = async function (pkgList) {
-  const MAVEN_CENTRAL_URL = "https://repo1.maven.org/maven2/";
+  const MAVEN_CENTRAL_URL =
+    process.env.MAVEN_CENTRAL_URL || "https://repo1.maven.org/maven2/";
   const ANDROID_MAVEN = "https://maven.google.com/";
   const cdepList = [];
   if (!pkgList || !pkgList.length) {
@@ -3285,14 +3288,15 @@ export const getCratesMetadata = async function (pkgList) {
  * @param {Array} pkgList Package list
  */
 export const getDartMetadata = async function (pkgList) {
-  const PUB_DEV_URL = "https://pub.dev/api/packages/";
+  const PUB_DEV_URL = process.env.PUB_DEV_URL || "https://pub.dev";
+  const PUB_PACKAGES_URL = PUB_DEV_URL + "/api/packages/";
   const cdepList = [];
   for (const p of pkgList) {
     try {
       if (DEBUG_MODE) {
-        console.log(`Querying pub.dev for ${p.name}`);
+        console.log(`Querying ${PUB_DEV_URL} for ${p.name}`);
       }
-      const res = await cdxgenAgent.get(PUB_DEV_URL + p.name, {
+      const res = await cdxgenAgent.get(PUB_PACKAGES_URL + p.name, {
         responseType: "json",
         headers: {
           Accept: "application/vnd.pub.v2+json"
@@ -3310,7 +3314,7 @@ export const getDartMetadata = async function (pkgList) {
             if (pubspec.homepage) {
               p.homepage = { url: pubspec.homepage };
             }
-            p.license = "https://pub.dev/packages/" + p.name + "/license";
+            p.license = `${PUB_DEV_URL}/packages/${p.name}/license`;
             cdepList.push(p);
             break;
           }
@@ -4764,49 +4768,99 @@ export const convertOSQueryResults = function (
   const pkgList = [];
   if (results && results.length) {
     for (const res of results) {
-      if (res.version) {
-        const version = res.version;
-        let name = res.name || res.device_id;
-        const group = "";
-        const subpath = res.path || res.admindir || res.source;
-        const publisher = res.maintainer || res.creator;
-        let scope = undefined;
-        const compScope = res.priority;
-        if (["required", "optional", "excluded"].includes(compScope)) {
-          scope = compScope;
-        }
-        const description =
-          res.description ||
-          res.arguments ||
-          res.device ||
-          res.codename ||
-          res.section ||
-          res.status ||
-          res.identifier ||
-          res.components;
-        // Re-use the name from query obj
-        if (!name && results.length === 1 && queryObj.name) {
-          name = queryObj.name;
-        }
-        if (name && version) {
-          const purl = new PackageURL(
-            queryObj.purlType || "swid",
-            group,
-            name,
-            version,
-            undefined,
-            subpath
-          );
-          pkgList.push({
-            name,
-            group,
-            version,
-            description,
-            publisher,
-            purl,
-            scope
-          });
+      const version =
+        res.version ||
+        res.hotfix_id ||
+        res.hardware_version ||
+        res.port ||
+        res.pid ||
+        res.subject_key_id ||
+        res.interface ||
+        res.instance_id;
+      let name =
+        res.name ||
+        res.device_id ||
+        res.hotfix_id ||
+        res.uuid ||
+        res.serial ||
+        res.pid ||
+        res.address ||
+        res.ami_id ||
+        res.interface ||
+        res.client_app_id;
+      let group = "";
+      const subpath = res.path || res.admindir || res.source;
+      let publisher =
+        res.publisher ||
+        res.maintainer ||
+        res.creator ||
+        res.manufacturer ||
+        res.provider ||
+        "";
+      if (publisher === "null") {
+        publisher = "";
+      }
+      let scope = undefined;
+      const compScope = res.priority;
+      if (["required", "optional", "excluded"].includes(compScope)) {
+        scope = compScope;
+      }
+      const description =
+        res.description ||
+        res.summary ||
+        res.arguments ||
+        res.device ||
+        res.codename ||
+        res.section ||
+        res.status ||
+        res.identifier ||
+        res.components ||
+        "";
+      // Re-use the name from query obj
+      if (!name && results.length === 1 && queryObj.name) {
+        name = queryObj.name;
+      }
+      let qualifiers = undefined;
+      if (res.identifying_number && res.identifying_number.length) {
+        qualifiers = {
+          tag_id: res.identifying_number.replace("{", "").replace("}", "")
+        };
+      }
+      if (name) {
+        name = name.replace(/ /g, "+").replace(/[:%]/g, "-");
+        group = group.replace(/ /g, "+").replace(/[:%]/g, "-");
+        const purl = new PackageURL(
+          queryObj.purlType || "swid",
+          group,
+          name,
+          version || "",
+          qualifiers,
+          subpath
+        ).toString();
+        const apkg = {
+          name,
+          group,
+          version: version || "",
+          description,
+          publisher,
+          "bom-ref": decodeURIComponent(purl),
+          purl,
+          scope,
+          type: queryObj.componentType
+        };
+        const props = [{ name: "cdx:osquery:category", value: queryCategory }];
+        for (const k of Object.keys(res).filter(
+          (p) => !["name", "version", "description", "publisher"].includes(p)
+        )) {
+          if (res[k] && res[k] !== "null") {
+            props.push({
+              name: k,
+              value: res[k]
+            });
+          }
         }
+        apkg.properties = props;
+        pkgList.push(apkg);
       }
     }
   }
@@ -5150,11 +5204,18 @@ export const collectJarNS = function (jarPath, pomPathMap = {}) {
   if (jarFiles && jarFiles.length) {
     for (const jf of jarFiles) {
       const jarname = jf;
-      const pomname =
+      let pomname =
         pomPathMap[basename(jf).replace(".jar", ".pom")] ||
         jarname.replace(".jar", ".pom");
       let pomData = undefined;
       let purl = undefined;
+      // In some cases, the pom name might be slightly different to the jar name
+      if (!existsSync(pomname)) {
+        const pomSearch = getAllFiles(dirname(jf), "*.pom");
+        if (pomSearch && pomSearch.length === 1) {
+          pomname = pomSearch[0];
+        }
+      }
       if (existsSync(pomname)) {
         pomData = parsePomXml(readFileSync(pomname, "utf-8"));
         if (pomData) {
@@ -5168,6 +5229,52 @@ export const collectJarNS = function (jarPath, pomPathMap = {}) {
           );
           purl = purlObj.toString();
         }
+      } else if (jf.includes(join(".m2", "repository"))) {
+        // Let's try our best to construct a purl for .m2 cache entries of the form
+        // .m2/repository/org/apache/logging/log4j/log4j-web/3.0.0-SNAPSHOT/log4j-web-3.0.0-SNAPSHOT.jar
+        const tmpA = jf.split(join(".m2", "repository", ""));
+        if (tmpA && tmpA.length) {
+          let tmpJarPath = tmpA[tmpA.length - 1];
+          // This would yield log4j-web-3.0.0-SNAPSHOT.jar
+          const jarFileName = basename(tmpJarPath).replace(".jar", "");
+          let tmpDirParts = dirname(tmpJarPath).split(_sep);
+          // Retrieve the version
+          let jarVersion = tmpDirParts.pop();
+          if (jarVersion === "plugins") {
+            jarVersion = tmpDirParts.pop();
+            if (jarVersion === "eclipse") {
+              jarVersion = tmpDirParts.pop();
+            }
+          }
+          // The result would form the group name
+          let jarGroupName = tmpDirParts.join(".").replace(/^\./, "");
+          let qualifierType = "jar";
+          // Support for p2 bundles and plugins
+          // See https://github.com/CycloneDX/cyclonedx-maven-plugin/issues/137
+          // See https://github.com/CycloneDX/cdxgen/pull/510#issuecomment-1702551615
+          if (jarGroupName.startsWith("p2.osgi.bundle")) {
+            jarGroupName = "p2.osgi.bundle";
+            qualifierType = "osgi-bundle";
+          } else if (jarGroupName.startsWith("p2.eclipse.plugin")) {
+            jarGroupName = "p2.eclipse.plugin";
+            qualifierType = "eclipse-plugin";
+          } else if (jarGroupName.startsWith("p2.binary")) {
+            jarGroupName = "p2.binary";
+            qualifierType = "eclipse-executable";
+          } else if (jarGroupName.startsWith("p2.org.eclipse.update.feature")) {
+            jarGroupName = "p2.org.eclipse.update.feature";
+            qualifierType = "eclipse-feature";
+          }
+          const purlObj = new PackageURL(
+            "maven",
+            jarGroupName,
+            jarFileName.replace(`-${jarVersion}`, ""),
+            jarVersion,
+            { type: qualifierType },
+            null
+          );
+          purl = purlObj.toString();
+        }
       } else if (jf.includes(join(".gradle", "caches"))) {
         // Let's try our best to construct a purl for gradle cache entries of the form
         // .gradle/caches/modules-2/files-2.1/org.xmlresolver/xmlresolver/4.2.0/f4dbdaa83d636dcac91c9003ffa7fb173173fe8d/xmlresolver-4.2.0-data.jar
@@ -5175,7 +5282,7 @@ export const collectJarNS = function (jarPath, pomPathMap = {}) {
         if (tmpA && tmpA.length) {
           let tmpJarPath = tmpA[tmpA.length - 1];
           // This would yield xmlresolver-4.2.0-data.jar
-          const jarFileName = basename(tmpJarPath);
+          const jarFileName = basename(tmpJarPath).replace(".jar", "");
           let tmpDirParts = dirname(tmpJarPath).split(_sep);
           // This would remove the hash from the end of the directory name
           tmpDirParts.pop();
@@ -5241,6 +5348,9 @@ export const convertJarNSToPackages = (jarNSMapping) => {
     }
     const name = pom.artifactId || purlObj.name;
     if (!name) {
+      console.warn(
+        `Unable to identify the metadata for ${purl}. This will be skipped.`
+      );
       continue;
     }
     const apackage = {
@@ -5783,27 +5893,37 @@ export const executeAtom = (src, args) => {
   }
   const freeMemoryGB = Math.floor(freemem() / 1024 / 1024 / 1024);
   if (DEBUG_MODE) {
-    console.log("Execuing", ATOM_BIN, args.join(" "));
+    console.log("Executing", ATOM_BIN, args.join(" "));
   }
   const env = {
     ...process.env,
     JAVA_OPTS: `-Xms${freeMemoryGB}G -Xmx${freeMemoryGB}G`
   };
+  env.PATH = `${env.PATH}${_delimiter}${join(
+    dirNameStr,
+    "node_modules",
+    ".bin"
+  )}`;
   const result = spawnSync(ATOM_BIN, args, {
     cwd,
     encoding: "utf-8",
     timeout: TIMEOUT_MS,
     env
   });
-  if (
-    result.stderr &&
-    result.stderr.includes(
-      "has been compiled by a more recent version of the Java Runtime"
-    )
-  ) {
-    console.log(
-      "Atom requires Java 17 or above. Please install a suitable version and re-run cdxgen to improve the SBoM accuracy.\nAlternatively, use the cdxgen container image."
-    );
+  if (result.stderr) {
+    if (
+      result.stderr.includes(
+        "has been compiled by a more recent version of the Java Runtime"
+      )
+    ) {
+      console.log(
+        "Atom requires Java 17 or above. Please install a suitable version and re-run cdxgen to improve the SBoM accuracy.\nAlternatively, use the cdxgen container image."
+      );
+    } else if (result.stderr.includes("astgen")) {
+      console.warn(
+        "WARN: Unable to locate astgen command. Install atom globally using sudo npm install -g @appthreat/atom to resolve this issue."
+      );
+    }
   }
   if (DEBUG_MODE) {
     if (result.stdout) {
@@ -5813,7 +5933,7 @@ export const executeAtom = (src, args) => {
       console.log(result.stderr);
     }
   }
-  return true;
+  return !result.error;
 };
 
 /**
@@ -6232,3 +6352,14 @@ export const addEvidenceForImports = (pkgList, allImports) => {
   }
   return pkgList;
 };
+
+export const componentSorter = (a, b) => {
+  if (a && b) {
+    for (const k of ["bom-ref", "purl", "name"]) {
+      if (a[k] && b[k]) {
+        return a[k].localeCompare(b[k]);
+      }
+    }
+  }
+  return a.localeCompare(b);
+};

package/utils.test.js

@@ -502,10 +502,10 @@ test("parse maven tree", () => {
     group: "com.pogeyan.cmis",
     name: "copper-server",
     version: "1.15.2",
-    qualifiers: { type: "jar" }
+    qualifiers: { type: "war" }
   });
   expect(parsedList.dependenciesList[0]).toEqual({
-    ref: "pkg:maven/com.pogeyan.cmis/copper-server@1.15.2?type=jar",
+    ref: "pkg:maven/com.pogeyan.cmis/copper-server@1.15.2?type=war",
     dependsOn: [
       "pkg:maven/javax/javaee-web-api@7.0?type=jar",
       "pkg:maven/org.apache.chemistry.opencmis/chemistry-opencmis-server-support@1.0.0?type=jar",
@@ -553,6 +553,35 @@ test("parse maven tree", () => {
       "pkg:maven/org.apache.geode/geode-core@1.1.1?type=jar"
     ]
   });
+  parsedList = parseMavenTree(
+    readFileSync("./test/data/mvn-p2-plugin.txt", {
+      encoding: "utf-8"
+    })
+  );
+  expect(parsedList.pkgList.length).toEqual(79);
+  expect(parsedList.pkgList[0]).toEqual({
+    group: "example.group",
+    name: "eclipse-repository",
+    version: "1.0.0-SNAPSHOT",
+    qualifiers: { type: "eclipse-repository" }
+  });
+  expect(parsedList.pkgList[4]).toEqual({
+    group: "p2.eclipse.plugin",
+    name: "com.ibm.icu",
+    version: "67.1.0.v20200706-1749",
+    qualifiers: { type: "eclipse-plugin" }
+  });
+  expect(parsedList.dependenciesList.length).toEqual(79);
+  expect(parsedList.dependenciesList[0]).toEqual({
+    ref: "pkg:maven/example.group/eclipse-repository@1.0.0-SNAPSHOT?type=eclipse-repository",
+    dependsOn: [
+      "pkg:maven/example.group/example-feature@0.1.0-SNAPSHOT?type=eclipse-feature",
+      "pkg:maven/example.group/example-feature-2@0.2.0-SNAPSHOT?type=eclipse-feature",
+      "pkg:maven/example.group/example-bundle@0.1.0-SNAPSHOT?type=eclipse-plugin",
+      "pkg:maven/example.group/org.tycho.demo.rootfiles@1.0.0?type=p2-installable-unit",
+      "pkg:maven/example.group/org.tycho.demo.rootfiles.win@1.0.0-SNAPSHOT?type=p2-installable-unit"
+    ]
+  });
 });
 
 // Slow test
@@ -1358,8 +1387,8 @@ test("parsePkgLock", async () => {
   });
   parsedList = await parsePkgLock("./test/data/package-lock-v2.json");
   deps = parsedList.pkgList;
-  expect(deps.length).toEqual(1467);
-  expect(parsedList.dependenciesList.length).toEqual(1280);
+  expect(deps.length).toEqual(5433);
+  expect(parsedList.dependenciesList.length).toEqual(1616);
   expect(deps[0]).toEqual({
     "bom-ref": "pkg:npm/flink-dashboard@2.0.0",
     group: "",