@cyclonedx/cdxgen 9.4.0 → 9.5.0

package/utils.js

@@ -1,5 +1,5 @@
 import { globSync } from "glob";
-import { tmpdir, platform, freemem } from "node:os";
+import { homedir, tmpdir, platform, freemem } from "node:os";
 import {
   dirname,
   sep as _sep,
@@ -11,6 +11,7 @@ import {
 import {
   existsSync,
   readFileSync,
+  lstatSync,
   mkdtempSync,
   rmSync,
   copyFileSync,
@@ -26,19 +27,19 @@ let url = import.meta.url;
 if (!url.startsWith("file://")) {
   url = new URL(`file://${import.meta.url}`).toString();
 }
-const dirName = import.meta ? dirname(fileURLToPath(url)) : __dirname;
+const dirNameStr = import.meta ? dirname(fileURLToPath(url)) : __dirname;
 
 const licenseMapping = JSON.parse(
-  readFileSync(join(dirName, "data", "lic-mapping.json"))
+  readFileSync(join(dirNameStr, "data", "lic-mapping.json"))
 );
 const vendorAliases = JSON.parse(
-  readFileSync(join(dirName, "data", "vendor-alias.json"))
+  readFileSync(join(dirNameStr, "data", "vendor-alias.json"))
 );
 const spdxLicenses = JSON.parse(
-  readFileSync(join(dirName, "data", "spdx-licenses.json"))
+  readFileSync(join(dirNameStr, "data", "spdx-licenses.json"))
 );
 const knownLicenses = JSON.parse(
-  readFileSync(join(dirName, "data", "known-licenses.json"))
+  readFileSync(join(dirNameStr, "data", "known-licenses.json"))
 );
 import { load } from "cheerio";
 import { load as _load } from "js-yaml";
@@ -49,18 +50,19 @@ import StreamZip from "node-stream-zip";
 import { parseEDNString } from "edn-data";
 import { PackageURL } from "packageurl-js";
 import { getTreeWithPlugin } from "./piptree.js";
+import iconv from "iconv-lite";
 
-const selfPJson = JSON.parse(readFileSync(join(dirName, "package.json")));
+const selfPJson = JSON.parse(readFileSync(join(dirNameStr, "package.json")));
 const _version = selfPJson.version;
 
 // Refer to contrib/py-modules.py for a script to generate this list
 // The script needs to be used once every few months to update this list
 const PYTHON_STD_MODULES = JSON.parse(
-  readFileSync(join(dirName, "data", "python-stdlib.json"))
+  readFileSync(join(dirNameStr, "data", "python-stdlib.json"))
 );
 // Mapping between modules and package names
 const PYPI_MODULE_PACKAGE_MAPPING = JSON.parse(
-  readFileSync(join(dirName, "data", "pypi-pkg-aliases.json"))
+  readFileSync(join(dirNameStr, "data", "pypi-pkg-aliases.json"))
 );
 
 // Debug mode flag
@@ -168,10 +170,24 @@ export function getLicenses(pkg, format = "xml") {
           } else if (l.startsWith("http")) {
             if (!l.includes("opensource.org")) {
               licenseContent.name = "CUSTOM";
+            } else {
+              const possibleId = l
+                .replace("http://www.opensource.org/licenses/", "")
+                .toUpperCase();
+              spdxLicenses.forEach((v) => {
+                if (v.toUpperCase() === possibleId) {
+                  licenseContent.id = v;
+                }
+              });
             }
             if (l.includes("mit-license")) {
               licenseContent.id = "MIT";
             }
+            // We always need a name to avoid validation errors
+            // Issue: #469
+            if (!licenseContent.name && !licenseContent.id) {
+              licenseContent.name = "CUSTOM";
+            }
             licenseContent.url = l;
           } else {
             licenseContent.name = l;
@@ -1449,7 +1465,7 @@ export const parseGradleDep = function (
     }
     let stack = [last_purl];
     const depRegex =
-      /^.*?--- +(?<group>[^\s:]+):(?<name>[^\s:]+)(?::(?:{strictly [[]?)?(?<versionspecified>[^,\s:}]+))?(?:})?(?:[^->]* +-> +(?<versionoverride>[^\s:]+))?/gm;
+      /^.*?--- +(?<groupspecified>[^\s:]+) ?:(?<namespecified>[^\s:]+)(?::(?:{strictly [[]?)?(?<versionspecified>[^,\s:}]+))?(?:})?(?:[^->]* +-> +(?:(?<groupoverride>[^\s:]+):(?<nameoverride>[^\s:]+):)?(?<versionoverride>[^\s:]+))?/gm;
     for (const rline of rawOutput.split("\n")) {
       if (!rline) {
         continue;
@@ -1467,16 +1483,9 @@ export const parseGradleDep = function (
         rline.startsWith("\\--- ")
       ) {
         last_level = 1;
-        if (rline.startsWith("+--- project :")) {
-          const tmpProj = rline.split("+--- project :");
-          last_project_purl = `pkg:maven/${tmpProj[1].trim()}@${rootProjectVersion}?type=jar`;
-          stack = [last_project_purl];
-          last_purl = last_project_purl;
-        } else {
-          last_project_purl = first_purl;
-          last_purl = last_project_purl;
-          stack = [first_purl];
-        }
+        last_project_purl = first_purl;
+        last_purl = last_project_purl;
+        stack = [first_purl];
       }
       if (rline.includes(" - ")) {
         profileName = rline.split(" - ")[0];
@@ -1489,76 +1498,86 @@ export const parseGradleDep = function (
         }
       }
       while ((match = depRegex.exec(rline))) {
-        const [line, group, name, versionspecified, versionoverride] = match;
+        const [
+          line,
+          groupspecified,
+          namespecified,
+          versionspecified,
+          groupoverride,
+          nameoverride,
+          versionoverride
+        ] = match;
+        const group = groupoverride || groupspecified;
+        const name = nameoverride || namespecified;
         const version = versionoverride || versionspecified;
-        const level = line.split(group)[0].length / 5;
-        if (version !== undefined) {
+        const level = line.split(groupspecified)[0].length / 5;
+        if (version !== undefined || group === "project") {
           let purlString = new PackageURL(
             "maven",
-            group,
+            group !== "project" ? group : rootProjectGroup,
             name,
-            version,
+            version !== undefined ? version : rootProjectVersion,
             { type: "jar" },
             null
           ).toString();
           purlString = decodeURIComponent(purlString);
           keys_cache[purlString + "_" + last_purl] = true;
-          if (group !== "project") {
-            // Filter duplicates
-            if (!deps_keys_cache[purlString]) {
-              deps_keys_cache[purlString] = true;
-              const adep = {
-                group,
-                name: name,
-                version: version,
-                qualifiers: { type: "jar" }
-              };
-              if (scope) {
-                adep["scope"] = scope;
-              }
-              if (profileName) {
-                adep.properties = [
-                  {
-                    name: "GradleProfileName",
-                    value: profileName
-                  }
-                ];
-              }
-              deps.push(adep);
+          // Filter duplicates
+          if (!deps_keys_cache[purlString]) {
+            deps_keys_cache[purlString] = true;
+            const adep = {
+              group: group !== "project" ? group : rootProjectGroup,
+              name: name,
+              version: version !== undefined ? version : rootProjectVersion,
+              qualifiers: { type: "jar" }
+            };
+            adep["purl"] = purlString;
+            adep["bom-ref"] = purlString;
+            if (scope) {
+              adep["scope"] = scope;
             }
-            if (!level_trees[purlString]) {
-              level_trees[purlString] = [];
+            if (profileName) {
+              adep.properties = [
+                {
+                  name: "GradleProfileName",
+                  value: profileName
+                }
+              ];
             }
-            if (level == 0) {
-              stack = [first_purl];
-              stack.push(purlString);
-            } else if (last_purl === "") {
-              stack.push(purlString);
-            } else if (level > last_level) {
-              const cnodes = level_trees[last_purl] || [];
-              if (!cnodes.includes(purlString)) {
-                cnodes.push(purlString);
-              }
-              level_trees[last_purl] = cnodes;
-              if (stack[stack.length - 1] !== purlString) {
-                stack.push(purlString);
-              }
-            } else {
-              for (let i = level; i <= last_level; i++) {
-                stack.pop();
-              }
-              const last_stack =
-                stack.length > 0 ? stack[stack.length - 1] : last_project_purl;
-              const cnodes = level_trees[last_stack] || [];
-              if (!cnodes.includes(purlString)) {
-                cnodes.push(purlString);
-              }
-              level_trees[last_stack] = cnodes;
+            deps.push(adep);
+          }
+          if (!level_trees[purlString]) {
+            level_trees[purlString] = [];
+          }
+          if (level == 0) {
+            stack = [first_purl];
+            stack.push(purlString);
+          } else if (last_purl === "") {
+            stack.push(purlString);
+          } else if (level > last_level) {
+            const cnodes = level_trees[last_purl] || [];
+            if (!cnodes.includes(purlString)) {
+              cnodes.push(purlString);
+            }
+            level_trees[last_purl] = cnodes;
+            if (stack[stack.length - 1] !== purlString) {
               stack.push(purlString);
             }
-            last_level = level;
-            last_purl = purlString;
+          } else {
+            for (let i = level; i <= last_level; i++) {
+              stack.pop();
+            }
+            const last_stack =
+              stack.length > 0 ? stack[stack.length - 1] : last_project_purl;
+            const cnodes = level_trees[last_stack] || [];
+            if (!cnodes.includes(purlString)) {
+              cnodes.push(purlString);
+            }
+            level_trees[last_stack] = cnodes;
+            stack.push(purlString);
           }
+          last_level = level;
+          last_purl = purlString;
         }
       }
     }
@@ -4118,13 +4137,19 @@ export const parseEdnData = function (rawEdnData) {
 };
 
 export const parseNupkg = async function (nupkgFile) {
-  const pkgList = [];
-  const pkg = { group: "" };
   let nuspecData = await readZipEntry(nupkgFile, ".nuspec");
-  // Remove byte order mark
-  if (nuspecData.charCodeAt(0) === 0xfeff) {
-    nuspecData = nuspecData.slice(1);
+  if (!nuspecData) {
+    return [];
   }
+  if (nuspecData.charCodeAt(0) === 65533) {
+    nuspecData = await readZipEntry(nupkgFile, ".nuspec", "ucs2");
+  }
+  return await parseNuspecData(nupkgFile, nuspecData);
+};
+
+export const parseNuspecData = async function (nupkgFile, nuspecData) {
+  const pkgList = [];
+  const pkg = { group: "" };
   let npkg = undefined;
   try {
     npkg = xml2js(nuspecData, {
@@ -4136,9 +4161,13 @@ export const parseNupkg = async function (nupkgFile) {
       commentKey: "value"
     }).package;
   } catch (e) {
-    // If we are parsing with invalid encoding unicode replacement character is used
+    // If we are parsing with invalid encoding, unicode replacement character is used
     if (nuspecData.charCodeAt(0) === 65533) {
       console.log(`Unable to parse ${nupkgFile} in utf-8 mode`);
+    } else {
+      console.log(
+        "Unable to parse this package. Tried utf-8 and ucs2 encoding."
+      );
     }
   }
   if (!npkg) {
@@ -4851,48 +4880,100 @@ export const parseSwiftResolved = (resolvedFile) => {
  *
  * @param {string} mavenCmd Maven command to use
  * @param {string} basePath Path to the maven project
+ * @param {boolean} cleanup Remove temporary directories
+ * @param {boolean} includeCacheDir Include maven and gradle cache directories
  */
-export const collectMvnDependencies = function (mavenCmd, basePath) {
-  const tempDir = mkdtempSync(join(tmpdir(), "mvn-deps-"));
-  console.log(
-    `Executing 'mvn dependency:copy-dependencies -DoutputDirectory=${tempDir} -DexcludeTransitive=true -DincludeScope=runtime' in ${basePath}`
-  );
-  const result = spawnSync(
-    mavenCmd,
-    [
-      "dependency:copy-dependencies",
-      `-DoutputDirectory=${tempDir}`,
-      "-DexcludeTransitive=true",
-      "-DincludeScope=runtime",
-      "-U",
-      "-Dmdep.prependGroupId=" + (process.env.MAVEN_PREPEND_GROUP || "false"),
-      "-Dmdep.stripVersion=" + (process.env.MAVEN_STRIP_VERSION || "false")
-    ],
-    { cwd: basePath, encoding: "utf-8" }
-  );
+export const collectMvnDependencies = function (
+  mavenCmd,
+  basePath,
+  cleanup = true,
+  includeCacheDir = false
+) {
   let jarNSMapping = {};
-  if (result.status !== 0 || result.error) {
-    console.error(result.stdout, result.stderr);
-    console.log(
-      "Resolve the above maven error. You can try the following remediation tips:\n"
-    );
+  const MAVEN_CACHE_DIR =
+    process.env.MAVEN_CACHE_DIR || join(homedir(), ".m2", "repository");
+  const tempDir = mkdtempSync(join(tmpdir(), "mvn-deps-"));
+  const copyArgs = [
+    "dependency:copy-dependencies",
+    `-DoutputDirectory=${tempDir}`,
+    "-U",
+    "-Dmdep.copyPom=true",
+    "-Dmdep.useRepositoryLayout=true",
+    "-Dmdep.includeScope=compile",
+    "-Dmdep.prependGroupId=" + (process.env.MAVEN_PREPEND_GROUP || "false"),
+    "-Dmdep.stripVersion=" + (process.env.MAVEN_STRIP_VERSION || "false")
+  ];
+  if (basePath && basePath !== MAVEN_CACHE_DIR) {
     console.log(
-      "1. Check if the correct version of maven is installed and available in the PATH."
+      `Executing '${mavenCmd} dependency:copy-dependencies ${copyArgs.join(
+        " "
+      )}' in ${basePath}`
     );
-    console.log(
-      "2. Perform 'mvn compile package' before invoking this command. Fix any errors found during this invocation."
+    const result = spawnSync(mavenCmd, copyArgs, {
+      cwd: basePath,
+      encoding: "utf-8"
+    });
+    if (result.status !== 0 || result.error) {
+      console.error(result.stdout, result.stderr);
+      console.log(
+        "Resolve the above maven error. You can try the following remediation tips:\n"
+      );
+      console.log(
+        "1. Check if the correct version of maven is installed and available in the PATH."
+      );
+      console.log(
+        "2. Perform 'mvn compile package' before invoking this command. Fix any errors found during this invocation."
+      );
+      console.log(
+        "3. Ensure the temporary directory is available and has sufficient disk space to copy all the artifacts."
+      );
+    } else {
+      jarNSMapping = collectJarNS(tempDir);
+    }
+  }
+  if (includeCacheDir || basePath === MAVEN_CACHE_DIR) {
+    // slow operation
+    jarNSMapping = collectJarNS(MAVEN_CACHE_DIR);
+  }
+
+  // Clean up
+  if (cleanup && tempDir && tempDir.startsWith(tmpdir()) && rmSync) {
+    rmSync(tempDir, { recursive: true, force: true });
+  }
+  return jarNSMapping;
+};
+
+export const collectGradleDependencies = (
+  gradleCmd,
+  basePath,
+  cleanup = true, // eslint-disable-line no-unused-vars
+  includeCacheDir = false // eslint-disable-line no-unused-vars
+) => {
+  // HELP WANTED: We need an init script that mimics maven copy-dependencies that only collects the project specific jars and poms
+  // Construct gradle cache directory
+  let GRADLE_CACHE_DIR =
+    process.env.GRADLE_CACHE_DIR ||
+    join(homedir(), ".gradle", "caches", "modules-2", "files-2.1");
+  if (process.env.GRADLE_USER_HOME) {
+    GRADLE_CACHE_DIR = join(
+      process.env.GRADLE_USER_HOME,
+      "caches",
+      "modules-2",
+      "files-2.1"
     );
+  }
+  if (DEBUG_MODE) {
+    console.log("Collecting jars from", GRADLE_CACHE_DIR);
     console.log(
-      "3. Ensure the temporary directory is available and has sufficient disk space to copy all the artifacts."
+      "To improve performance, ensure only the project dependencies are present in this cache location."
     );
-  } else {
-    jarNSMapping = collectJarNS(tempDir);
   }
-  // Clean up
-  if (tempDir && tempDir.startsWith(tmpdir()) && rmSync) {
-    console.log(`Cleaning up ${tempDir}`);
-    rmSync(tempDir, { recursive: true, force: true });
+  const pomPathMap = {};
+  const pomFiles = getAllFiles(GRADLE_CACHE_DIR, "**/*.pom");
+  for (const apom of pomFiles) {
+    pomPathMap[basename(apom)] = apom;
   }
+  const jarNSMapping = collectJarNS(GRADLE_CACHE_DIR, pomPathMap);
   return jarNSMapping;
 };
 
@@ -4900,10 +4981,11 @@ export const collectMvnDependencies = function (mavenCmd, basePath) {
  * Method to collect class names from all jars in a directory
  *
  * @param {string} jarPath Path containing jars
+ * @param {object} pomPathMap Map containing jar to pom names. Required to successful parse gradle cache.
  *
  * @return object containing jar name and class list
  */
-export const collectJarNS = function (jarPath) {
+export const collectJarNS = function (jarPath, pomPathMap = {}) {
   const jarNSMapping = {};
   console.log(
     `About to identify class names for all jars in the path ${jarPath}`
@@ -4912,31 +4994,72 @@ export const collectJarNS = function (jarPath) {
   const jarFiles = getAllFiles(jarPath, "**/*.jar");
   if (jarFiles && jarFiles.length) {
     for (const jf of jarFiles) {
-      const jarname = basename(jf);
+      const jarname = jf;
+      const pomname =
+        pomPathMap[basename(jf).replace(".jar", ".pom")] ||
+        jarname.replace(".jar", ".pom");
+      let pomData = undefined;
+      let purl = undefined;
+      if (existsSync(pomname)) {
+        pomData = parsePomXml(readFileSync(pomname, "utf-8"));
+        if (pomData) {
+          const purlObj = new PackageURL(
+            "maven",
+            pomData.groupId || "",
+            pomData.artifactId,
+            pomData.version,
+            { type: "jar" },
+            null
+          );
+          purl = purlObj.toString();
+        }
+      } else if (jf.includes(join(".gradle", "caches"))) {
+        // Let's try our best to construct a purl for gradle cache entries of the form
+        // .gradle/caches/modules-2/files-2.1/org.xmlresolver/xmlresolver/4.2.0/f4dbdaa83d636dcac91c9003ffa7fb173173fe8d/xmlresolver-4.2.0-data.jar
+        const tmpA = jf.split(join("files-2.1", ""));
+        if (tmpA && tmpA.length) {
+          let tmpJarPath = tmpA[tmpA.length - 1];
+          // This would yield xmlresolver-4.2.0-data.jar
+          const jarFileName = basename(tmpJarPath);
+          let tmpDirParts = dirname(tmpJarPath).split(_sep);
+          // This would remove the hash from the end of the directory name
+          tmpDirParts.pop();
+          // Retrieve the version
+          const jarVersion = tmpDirParts.pop();
+          // The result would form the group name
+          const jarGroupName = tmpDirParts.join(".").replace(/^\./, "");
+          const purlObj = new PackageURL(
+            "maven",
+            jarGroupName,
+            jarFileName.replace(`-${jarVersion}`, ""),
+            jarVersion,
+            { type: "jar" },
+            null
+          );
+          purl = purlObj.toString();
+        }
+      }
       if (DEBUG_MODE) {
         console.log(`Executing 'jar tf ${jf}'`);
       }
       const jarResult = spawnSync("jar", ["-tf", jf], { encoding: "utf-8" });
-      if (jarResult.status !== 0) {
-        console.error(jarResult.stdout, jarResult.stderr);
-        console.log(
-          "Check if JRE is installed and the jar command is available in the PATH."
-        );
-        break;
-      } else {
-        const consolelines = (jarResult.stdout || "").split("\n");
-        const nsList = consolelines
-          .filter((l) => {
-            return l.includes(".class") && !l.includes("-INF");
-          })
-          .map((e) => {
-            return e
-              .replace(/\/$/, "")
-              .replace(/\//g, ".")
-              .replace(".class", "");
-          });
-        jarNSMapping[jarname] = nsList;
-      }
+      const consolelines = (jarResult.stdout || "").split("\n");
+      const nsList = consolelines
+        .filter((l) => {
+          return (
+            l.includes(".class") &&
+            !l.includes("-INF") &&
+            !l.includes("module-info")
+          );
+        })
+        .map((e) => {
+          return e.replace(".class", "").replace(/\/$/, "").replace(/\//g, ".");
+        });
+      jarNSMapping[purl || jf] = {
+        jarFile: jf,
+        pom: pomData,
+        namespaces: nsList
+      };
     }
     if (!jarNSMapping) {
       console.log(`Unable to determine class names for the jars in ${jarPath}`);
@@ -4944,12 +5067,69 @@ export const collectJarNS = function (jarPath) {
   } else {
     console.log(`${jarPath} did not contain any jars.`);
   }
-  if (DEBUG_MODE) {
-    console.log("JAR Namespace mapping", jarNSMapping);
-  }
   return jarNSMapping;
 };
 
+export const convertJarNSToPackages = (jarNSMapping) => {
+  let pkgList = [];
+  for (const purl of Object.keys(jarNSMapping)) {
+    let { jarFile, pom, namespaces } = jarNSMapping[purl];
+    if (!pom) {
+      pom = {};
+    }
+    let purlObj = undefined;
+    try {
+      purlObj = PackageURL.fromString(purl);
+    } catch (e) {
+      // ignore
+      purlObj = {};
+    }
+    const name = pom.artifactId || purlObj.name;
+    if (!name) {
+      continue;
+    }
+    const apackage = {
+      name,
+      group: pom.groupId || purlObj.namespace || "",
+      version: pom.version || purlObj.version,
+      description: (pom.description || "").trim(),
+      purl,
+      "bom-ref": purl,
+      evidence: {
+        identity: {
+          field: "purl",
+          confidence: 0.5,
+          methods: [
+            {
+              technique: "filename",
+              confidence: 1,
+              value: jarFile
+            }
+          ]
+        }
+      },
+      properties: [
+        {
+          name: "SrcFile",
+          value: jarFile
+        },
+        {
+          name: "Namespaces",
+          value: namespaces.join("\n")
+        }
+      ]
+    };
+    if (pom.url) {
+      apackage["homepage"] = { url: pom.url };
+    }
+    if (pom.scm) {
+      apackage["repository"] = { url: pom.scm };
+    }
+    pkgList.push(apackage);
+  }
+  return pkgList;
+};
+
 export const parsePomXml = function (pomXmlData) {
   if (!pomXmlData) {
     return undefined;
@@ -5293,10 +5473,15 @@ export const sbtPluginsPath = function (projectPath) {
  *
  * @param {string} zipFile Zip file to read
  * @param {string} filePattern File pattern
+ * @param {string} contentEncoding Encoding. Defaults to utf-8
  *
  * @returns File contents
  */
-export const readZipEntry = async function (zipFile, filePattern) {
+export const readZipEntry = async function (
+  zipFile,
+  filePattern,
+  contentEncoding = "utf-8"
+) {
   let retData = undefined;
   try {
     const zip = new StreamZip.async({ file: zipFile });
@@ -5311,7 +5496,7 @@ export const readZipEntry = async function (zipFile, filePattern) {
       }
       if (entry.name.endsWith(filePattern)) {
         const fileData = await zip.entryData(entry.name);
-        retData = Buffer.from(fileData).toString();
+        retData = iconv.decode(Buffer.from(fileData), contentEncoding);
         break;
       }
     }
@@ -5418,7 +5603,7 @@ export const getAtomCommand = () => {
   }
   const NODE_CMD = process.env.NODE_CMD || "node";
   const localAtom = join(
-    dirName,
+    dirNameStr,
     "node_modules",
     "@appthreat",
     "atom",
@@ -5431,6 +5616,8 @@ export const getAtomCommand = () => {
 };
 
 export const executeAtom = (src, args) => {
+  let cwd =
+    existsSync(src) && lstatSync(src).isDirectory() ? src : dirname(src);
   let ATOM_BIN = getAtomCommand();
   if (ATOM_BIN.includes(" ")) {
     const tmpA = ATOM_BIN.split(" ");
@@ -5448,7 +5635,7 @@ export const executeAtom = (src, args) => {
     JAVA_OPTS: `-Xms${freeMemoryGB}G -Xmx${freeMemoryGB}G`
   };
   const result = spawnSync(ATOM_BIN, args, {
-    cwd: src,
+    cwd,
     encoding: "utf-8",
     timeout: TIMEOUT_MS,
     env