@cyclonedx/cdxgen 8.4.13 → 8.5.0

package/README.md

@@ -53,7 +53,7 @@ NOTE:
 Footnotes:
 
 - [1] - For multi-module application, the BoM file could include components that may not be included in the packaged war or ear file.
-- [2] - Pip freeze is automatically performed to improve precision.
+- [2] - Pip freeze is automatically performed to improve precision. Requires virtual environment.
 - [3] - Perform dotnet or nuget restore to generate project.assets.json. Without this file cdxgen would not include indirect dependencies.
 - [4] - See section on plugins
 - [5] - Powered by osquery. See section on plugins
@@ -254,7 +254,7 @@ cdxgen can retain the dependency tree under the `dependencies` attribute for a s
 | MVN_ARGS                     | Set to pass additional arguments such as profile or settings to maven                                                       |
 | MAVEN_HOME                   | Specify maven home                                                                                                          |
 | GRADLE_CACHE_DIR             | Specify gradle cache directory. Useful for class name resolving                                                             |
-| GRADLE_MULTI_PROJECT_MODE    | Set this variable for gradle multi-project applications. Do not use this with recurse mode.                                 |
+| GRADLE_MULTI_PROJECT_MODE    | Unused. Automatically handled                                                                                               |
 | GRADLE_ARGS                  | Set to pass additional arguments such as profile or settings to gradle. Eg: --configuration runtimeClassPath                |
 | GRADLE_HOME                  | Specify gradle home                                                                                                         |
 | GRADLE_CMD                   | Set to override gradle command                                                                                              |

package/docker.test.js

@@ -1,5 +1,5 @@
 const dockerLib = require("./docker");
-const { jest, expect, test } = require("@jest/globals");
+const { expect, test } = require("@jest/globals");
 
 test("docker connection", async () => {
   const dockerConn = await dockerLib.getConnection();
@@ -58,16 +58,14 @@ test("parseImageName tests", () => {
 });
 
 test("docker getImage", async () => {
-  jest.setTimeout(120000);
   const imageData = await dockerLib.getImage("hello-world:latest");
   if (imageData) {
     const removeData = await dockerLib.removeImage("hello-world:latest");
     expect(removeData);
   }
-});
+}, 120000);
 
 test("docker getImage", async () => {
-  jest.setTimeout(120000);
   const imageData = await dockerLib.exportImage("hello-world:latest");
   expect(imageData);
-});
+}, 120000);

package/index.js

@@ -248,6 +248,11 @@ function addMetadata(parentComponent = {}, format = "xml", options = {}) {
       if (format == "xml" && firstPComp.component) {
         metadata.component = firstPComp.component;
       } else {
+        // Retain the components of parent component
+        // Bug #317 fix
+        if (parentComponent && parentComponent.components) {
+          firstPComp.components = parentComponent.components;
+        }
         metadata.component = firstPComp;
       }
     } else {
@@ -1158,268 +1163,154 @@ const createJavaBom = async (path, options) => {
       path,
       (options.multiProject ? "**/" : "") + "build.gradle*"
     );
+    let allProjects = [];
+    const allProjectsAddedPurls = [];
+    const rootDependsOn = [];
+    // Execute gradle properties
+    if (gradleFiles && gradleFiles.length) {
+      let retMap = utils.executeGradleProperties(path, null, null);
+      const allProjectsStr = retMap.projects || [];
+      let rootProject = retMap.rootProject;
+      if (rootProject) {
+        parentComponent = {
+          name: rootProject,
+          type: "application",
+          qualifiers: { type: "jar" },
+          ...(retMap.metadata || {})
+        };
+      }
+      // Get the sub-project properties and set the root dependencies
+      if (allProjectsStr && allProjectsStr.length) {
+        for (let spstr of allProjectsStr) {
+          retMap = utils.executeGradleProperties(path, null, spstr);
+          let rootSubProject = retMap.rootProject;
+          if (rootSubProject) {
+            let rspName = rootSubProject.replace(/^:/, "").replace(/:/, "/");
+            const rootSubProjectObj = {
+              name: rspName,
+              type: "application",
+              qualifiers: { type: "jar" },
+              ...(retMap.metadata || {})
+            };
+            const rootSubProjectPurl = decodeURIComponent(
+              new PackageURL(
+                "maven",
+                rootSubProjectObj.group || parentComponent.group || "",
+                rootSubProjectObj.name,
+                rootSubProjectObj.version,
+                rootSubProjectObj.qualifiers,
+                null
+              ).toString()
+            );
+            rootSubProjectObj["purl"] = rootSubProjectPurl;
+            rootSubProjectObj["bom-ref"] = rootSubProjectPurl;
+            if (!allProjectsAddedPurls.includes(rootSubProjectPurl)) {
+              allProjects.push(rootSubProjectObj);
+              rootDependsOn.push(rootSubProjectPurl);
+              allProjectsAddedPurls.push(rootSubProjectPurl);
+            }
+          }
+        }
+        // Bug #317 fix
+        parentComponent.components = allProjects.flatMap((s) => {
+          delete s.qualifiers;
+          return s;
+        });
+        dependencies.push({
+          ref: decodeURIComponent(
+            new PackageURL(
+              "maven",
+              parentComponent.group,
+              parentComponent.name,
+              parentComponent.version,
+              parentComponent.qualifiers,
+              null
+            ).toString()
+          ),
+          dependsOn: rootDependsOn
+        });
+      }
+    }
     if (gradleFiles && gradleFiles.length && options.installDeps) {
       let gradleCmd = utils.getGradleCommand(path, null);
-      const multiProjectMode = process.env.GRADLE_MULTI_PROJECT_MODE || "";
-      // Support for multi-project applications
-      // Let's experiment with defaulting to multi-project mode when multiple gradle files gets detected
-      if (
-        ["true", "1"].includes(multiProjectMode) ||
-        (gradleFiles.length > 1 && !["false", "0"].includes(multiProjectMode))
-      ) {
-        let gradleProjectsArgs = ["projects", "-q", "--console", "plain"];
+      if (!allProjects || !allProjects.length) {
+        allProjects.push(parentComponent);
+      }
+      for (let sp of allProjects) {
+        let gradleDepArgs = [
+          sp.name === parentComponent.name
+            ? "dependencies"
+            : `:${sp.name}:dependencies`,
+          "-q",
+          "--console",
+          "plain",
+          "--build-cache"
+        ];
+        // Support custom GRADLE_ARGS such as --configuration runtimeClassPath
         if (process.env.GRADLE_ARGS) {
           const addArgs = process.env.GRADLE_ARGS.split(" ");
-          gradleProjectsArgs = gradleProjectsArgs.concat(addArgs);
+          gradleDepArgs = gradleDepArgs.concat(addArgs);
         }
         console.log(
           "Executing",
           gradleCmd,
-          gradleProjectsArgs.join(" "),
-          "projects in",
+          gradleDepArgs.join(" "),
+          "in",
           path
         );
-        const result = spawnSync(gradleCmd, gradleProjectsArgs, {
+        const sresult = spawnSync(gradleCmd, gradleDepArgs, {
           cwd: path,
           encoding: "utf-8",
           timeout: TIMEOUT_MS
         });
-        if (result.status !== 0 || result.error) {
-          if (result.stderr) {
-            console.error(result.stdout, result.stderr);
-          }
-          console.log(
-            "1. Check if the correct version of java and gradle are installed and available in PATH. For example, some project might require Java 11 with gradle 7.\n cdxgen container image bundles Java 17 with gradle 8 which might be incompatible."
-          );
-          options.failOnError && process.exit(1);
-        }
-        const stdout = result.stdout;
-        if (stdout) {
-          const cmdOutput = Buffer.from(stdout).toString();
-          const retMap = utils.parseGradleProjects(cmdOutput);
-          const allProjects = retMap.projects || [];
-          let rootProject = retMap.rootProject;
-          if (rootProject) {
-            parentComponent = {
-              group: "",
-              name: rootProject,
-              version: "latest",
-              type: "maven",
-              qualifiers: { type: "jar" }
-            };
-          }
-          if (!allProjects) {
-            console.log(
-              "No projects found. Is this a gradle multi-project application?"
-            );
-            options.failOnError && process.exit(1);
-          } else {
-            console.log(
-              "Found",
-              allProjects.length,
-              "gradle sub-projects. This might take a while ..."
-            );
-            // We need the first dependency between the root project and child projects
-            // See: #249 and #315
-            const rootDependsOn = [];
-            for (let sp of allProjects) {
-              sp = sp.replace(":", "");
-              rootDependsOn.push(
-                decodeURIComponent(
-                  new PackageURL(
-                    "maven",
-                    "",
-                    sp,
-                    parentComponent.version,
-                    parentComponent.qualifiers,
-                    null
-                  ).toString()
-                )
-              );
-            }
-            dependencies.push({
-              ref: decodeURIComponent(
-                new PackageURL(
-                  "maven",
-                  parentComponent.group,
-                  parentComponent.name,
-                  parentComponent.version,
-                  parentComponent.qualifiers,
-                  null
-                ).toString()
-              ),
-              dependsOn: rootDependsOn
-            });
-            for (let sp of allProjects) {
-              let gradleDepArgs = [
-                sp + ":dependencies",
-                "-q",
-                "--console",
-                "plain"
-              ];
-              // Support custom GRADLE_ARGS such as --configuration runtimeClassPath
-              if (process.env.GRADLE_ARGS) {
-                const addArgs = process.env.GRADLE_ARGS.split(" ");
-                gradleDepArgs = gradleDepArgs.concat(addArgs);
-              }
-              console.log(
-                "Executing",
-                gradleCmd,
-                gradleDepArgs.join(" "),
-                "in",
-                path
-              );
-              const sresult = spawnSync(gradleCmd, gradleDepArgs, {
-                cwd: path,
-                encoding: "utf-8",
-                timeout: TIMEOUT_MS
-              });
-              if (sresult.status !== 0 || sresult.error) {
-                if (options.failOnError || DEBUG_MODE) {
-                  console.error(sresult.stdout, sresult.stderr);
-                }
-                options.failOnError && process.exit(1);
-              }
-              const sstdout = sresult.stdout;
-              if (sstdout) {
-                sp = sp.replace(":", "");
-                const cmdOutput = Buffer.from(sstdout).toString();
-                const parsedList = utils.parseGradleDep(cmdOutput, sp);
-                const dlist = parsedList.pkgList;
-                // Do not overwrite the parentComponent in multi-project mode
-                if (!parentComponent || !Object.keys(parentComponent).length) {
-                  parentComponent = dlist.splice(0, 1)[0];
-                }
-                if (
-                  parsedList.dependenciesList &&
-                  parsedList.dependenciesList
-                ) {
-                  dependencies = mergeDependencies(
-                    dependencies,
-                    parsedList.dependenciesList
-                  );
-                }
-                if (dlist && dlist.length) {
-                  if (DEBUG_MODE) {
-                    console.log(
-                      "Found",
-                      dlist.length,
-                      "packages in gradle project",
-                      sp
-                    );
-                  }
-                  pkgList = pkgList.concat(dlist);
-                } else {
-                  if (options.failOnError || DEBUG_MODE) {
-                    console.log("No packages were found in gradle project", sp);
-                  }
-                  options.failOnError && process.exit(1);
-                }
-              }
-            }
-            if (pkgList.length) {
-              console.log(
-                "Obtained",
-                pkgList.length,
-                "from this gradle multi-project. De-duping this list ..."
-              );
-            } else {
-              console.log(
-                "No packages found. Unset the environment variable GRADLE_MULTI_PROJECT_MODE and try again."
-              );
-              options.failOnError && process.exit(1);
-            }
+        if (sresult.status !== 0 || sresult.error) {
+          if (options.failOnError || DEBUG_MODE) {
+            console.error(sresult.stdout, sresult.stderr);
           }
-        } else {
-          console.error("Gradle unexpectedly didn't return any output");
           options.failOnError && process.exit(1);
         }
-      } else {
-        let gradleDepArgs = ["dependencies", "-q", "--console", "plain"];
-        // Support for overriding the gradle task name. Issue# 90
-        if (process.env.GRADLE_DEPENDENCY_TASK) {
-          gradleDepArgs = process.env.GRADLE_DEPENDENCY_TASK.split(" ");
-        } else if (process.env.GRADLE_ARGS) {
-          // Support custom GRADLE_ARGS such as --configuration runtimeClassPath
-          const addArgs = process.env.GRADLE_ARGS.split(" ");
-          gradleDepArgs = gradleDepArgs.concat(addArgs);
-        }
-        for (let f of gradleFiles) {
-          const basePath = pathLib.dirname(f);
-          // Fixes #157. Look for wrapper script in the nested directory
-          gradleCmd = utils.getGradleCommand(basePath, path);
-          console.log(
-            "Executing",
-            gradleCmd,
-            gradleDepArgs.join(" "),
-            "in",
-            basePath
+        const sstdout = sresult.stdout;
+        if (sstdout) {
+          const cmdOutput = Buffer.from(sstdout).toString();
+          const parsedList = utils.parseGradleDep(
+            cmdOutput,
+            sp.group,
+            sp.name,
+            sp.version
           );
-          const result = spawnSync(gradleCmd, gradleDepArgs, {
-            cwd: basePath,
-            encoding: "utf-8",
-            timeout: TIMEOUT_MS
-          });
-          if (result.status !== 0 || result.error) {
-            if (result.stderr) {
-              const cmdError = Buffer.from(result.stderr).toString();
-              if (
-                cmdError.includes(
-                  "is not part of the build defined by settings file"
-                ) ||
-                cmdError.includes(
-                  "was not found in any of the following sources"
-                )
-              ) {
-                console.log(
-                  "This is a multi-project gradle application. Set the environment variable GRADLE_MULTI_PROJECT_MODE to true to improve SBoM accuracy."
-                );
-              }
-              if (DEBUG_MODE) {
-                console.log("-----------------------");
-              }
-              console.error(result.stdout, result.stderr);
-              if (DEBUG_MODE) {
-                console.log("-----------------------");
-              }
-              options.failOnError && process.exit(1);
-            }
-
-            if (DEBUG_MODE || !result.stderr || options.failOnError) {
-              console.log(
-                "1. Check if the correct version of java and gradle are installed and available in PATH. For example, some project might require Java 11 with gradle 7.\n cdxgen container image bundles Java 17 with gradle 8 which might be incompatible."
-              );
-              console.log(
-                "2. When using tools such as sdkman, the init script must be invoked to set the PATH variables correctly."
-              );
-              options.failOnError && process.exit(1);
-            }
+          const dlist = parsedList.pkgList;
+          if (parsedList.dependenciesList && parsedList.dependenciesList) {
+            dependencies = mergeDependencies(
+              dependencies,
+              parsedList.dependenciesList
+            );
           }
-          const stdout = result.stdout;
-          if (stdout) {
-            const cmdOutput = Buffer.from(stdout).toString();
-            const parsedList = utils.parseGradleDep(cmdOutput);
-            const dlist = parsedList.pkgList;
-            if (!parentComponent || !Object.keys(parentComponent).length) {
-              parentComponent = dlist.splice(0, 1)[0];
-            }
-            if (parsedList.dependenciesList && parsedList.dependenciesList) {
-              dependencies = mergeDependencies(
-                dependencies,
-                parsedList.dependenciesList
-              );
-            }
-            if (dlist && dlist.length) {
-              pkgList = pkgList.concat(dlist);
-            } else {
+          if (dlist && dlist.length) {
+            if (DEBUG_MODE) {
               console.log(
-                "No packages were detected. If this is a multi-project gradle application set the environment variable GRADLE_MULTI_PROJECT_MODE to true and try again."
+                "Found",
+                dlist.length,
+                "packages in gradle project",
+                sp.name
               );
-              options.failOnError && process.exit(1);
             }
+            pkgList = pkgList.concat(dlist);
           }
         }
+      } // for
+      if (pkgList.length) {
+        console.log(
+          "Obtained",
+          pkgList.length,
+          "from this gradle multi-project. De-duping this list ..."
+        );
+      } else {
+        console.log(
+          "No packages found. Set the environment variable 'CDXGEN_DEBUG_MODE=debug' to troubleshoot any gradle related errors."
+        );
+        options.failOnError && process.exit(1);
       }
+
       pkgList = await utils.getMvnMetadata(pkgList);
       // Should we attempt to resolve class names
       if (options.resolveClass) {
@@ -2030,7 +1921,7 @@ const createPythonBom = async (path, options) => {
   );
   const reqFiles = utils.getAllFiles(
     path,
-    (options.multiProject ? "**/" : "") + "*requirements.txt"
+    (options.multiProject ? "**/" : "") + "*requirements*.txt"
   );
   const reqDirFiles = utils.getAllFiles(
     path,
@@ -2102,7 +1993,7 @@ const createPythonBom = async (path, options) => {
       }
     }
   }
-  if (requirementsMode || pipenvMode || setupPyMode) {
+  if (requirementsMode || pipenvMode) {
     if (pipenvMode) {
       spawnSync("pipenv", ["install"], { cwd: path, encoding: "utf-8" });
       const piplockFile = pathLib.join(path, "Pipfile.lock");
@@ -2119,61 +2010,77 @@ const createPythonBom = async (path, options) => {
     } else if (requirementsMode) {
       metadataFilename = "requirements.txt";
       if (reqFiles && reqFiles.length) {
+        let pipWarningShown = false;
         for (let f of reqFiles) {
           const basePath = pathLib.dirname(f);
           let reqData = undefined;
-          // Attempt to pip freeze to improve precision
+          let frozenMode = false;
+          // Attempt to pip freeze to improve precision. First try in venv mode
           if (options.installDeps) {
-            const result = spawnSync(PIP_CMD, ["freeze", "-r", f, "-l"], {
-              cwd: basePath,
-              encoding: "utf-8",
-              timeout: TIMEOUT_MS
-            });
+            const result = spawnSync(
+              PIP_CMD,
+              ["freeze", "-r", f, "-l", "--require-virtualenv"],
+              {
+                cwd: basePath,
+                encoding: "utf-8",
+                timeout: TIMEOUT_MS
+              }
+            );
             if (result.status === 0 && result.stdout) {
               reqData = Buffer.from(result.stdout).toString();
+              const dlist = await utils.parseReqFile(reqData, false);
+              if (dlist && dlist.length) {
+                pkgList = pkgList.concat(dlist);
+              }
+              frozenMode = true;
+            } else if (result.status !== 0 || result.error) {
+              if (DEBUG_MODE && !pipWarningShown) {
+                pipWarningShown = true;
+                console.log(
+                  "NOTE: Setup and activate a python virtual environment for this project prior to invoking cdxgen to improve SBoM accuracy."
+                );
+              }
             }
           }
-          // Fallback to parsing requirements file
-          if (!reqData) {
+          // Fallback to parsing manually
+          if (!frozenMode) {
             if (DEBUG_MODE) {
               console.log(
-                `Falling back to manually parsing ${f}. The result would be incomplete!`
+                `Manually parsing ${f}. The result would include only direct dependencies.`
               );
             }
             reqData = fs.readFileSync(f, { encoding: "utf-8" });
+            const dlist = await utils.parseReqFile(reqData, true);
+            if (dlist && dlist.length) {
+              pkgList = pkgList.concat(dlist);
+            }
           }
-          const dlist = await utils.parseReqFile(reqData);
-          if (dlist && dlist.length) {
-            pkgList = pkgList.concat(dlist);
-          }
-        }
+        } // for
         metadataFilename = reqFiles.join(", ");
       } else if (reqDirFiles && reqDirFiles.length) {
         for (let j in reqDirFiles) {
           const f = reqDirFiles[j];
           const reqData = fs.readFileSync(f, { encoding: "utf-8" });
-          const dlist = await utils.parseReqFile(reqData);
+          const dlist = await utils.parseReqFile(reqData, false);
           if (dlist && dlist.length) {
             pkgList = pkgList.concat(dlist);
           }
         }
         metadataFilename = reqDirFiles.join(", ");
       }
-    } else if (setupPyMode) {
-      const setupPyData = fs.readFileSync(setupPy, { encoding: "utf-8" });
-      dlist = await utils.parseSetupPyFile(setupPyData);
-      if (dlist && dlist.length) {
-        pkgList = pkgList.concat(dlist);
-      }
     }
   }
-  if (pkgList.length) {
-    return buildBomNSData(options, pkgList, "pypi", {
-      src: path,
-      filename: metadataFilename
-    });
+  if (!pkgList.length && setupPyMode) {
+    const setupPyData = fs.readFileSync(setupPy, { encoding: "utf-8" });
+    dlist = await utils.parseSetupPyFile(setupPyData);
+    if (dlist && dlist.length) {
+      pkgList = pkgList.concat(dlist);
+    }
   }
-  return {};
+  return buildBomNSData(options, pkgList, "pypi", {
+    src: path,
+    filename: metadataFilename
+  });
 };
 
 /**
@@ -3712,7 +3619,12 @@ const createMultiXBom = async (pathList, options) => {
       console.log("Scanning", path);
     }
     bomData = await createNodejsBom(path, options);
-    if (bomData && bomData.bomJson && bomData.bomJson.components) {
+    if (
+      bomData &&
+      bomData.bomJson &&
+      bomData.bomJson.components &&
+      bomData.bomJson.components.length
+    ) {
       if (DEBUG_MODE) {
         console.log(
           `Found ${bomData.bomJson.components.length} npm packages at ${path}`
@@ -3976,7 +3888,12 @@ const createMultiXBom = async (pathList, options) => {
       );
     }
     bomData = await createSwiftBom(path, options);
-    if (bomData && bomData.bomJson && bomData.bomJson.components) {
+    if (
+      bomData &&
+      bomData.bomJson &&
+      bomData.bomJson.components &&
+      bomData.bomJson.components.length
+    ) {
       if (DEBUG_MODE) {
         console.log(
           `Found ${bomData.bomJson.components.length} Swift packages at ${path}`
@@ -4088,7 +4005,7 @@ const createXBom = async (path, options) => {
   const poetryMode = fs.existsSync(pathLib.join(path, "poetry.lock"));
   const reqFiles = utils.getAllFiles(
     path,
-    (options.multiProject ? "**/" : "") + "*requirements.txt"
+    (options.multiProject ? "**/" : "") + "*requirements*.txt"
   );
   const reqDirFiles = utils.getAllFiles(
     path,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cyclonedx/cdxgen",
-  "version": "8.4.13",
+  "version": "8.5.0",
   "description": "Creates CycloneDX Software Bill-of-Materials (SBOM) from source or container image",
   "homepage": "http://github.com/cyclonedx/cdxgen",
   "author": "Prabhu Subramanian <prabhu@appthreat.com>",