@cyclonedx/cdxgen 9.8.6 → 9.8.8

package/index.js

@@ -60,6 +60,7 @@ import {
   parseGoVersionData,
   parseGosumData,
   parseGoListDep,
+  parseGoModGraph,
   parseGoModWhy,
   parseGoModData,
   parseGopkgData,
@@ -92,6 +93,7 @@ import {
   parseCsPkgLockData,
   parseCsPkgData,
   parseCsProjData,
+  parsePaketLockData,
   DEBUG_MODE,
   parsePyProjectToml,
   addEvidenceForImports,
@@ -2308,24 +2310,36 @@ export const createPythonBom = async (path, options) => {
     for (const f of poetryFiles) {
       const basePath = dirname(f);
       const lockData = readFileSync(f, { encoding: "utf-8" });
-      const dlist = await parsePoetrylockData(lockData, f);
-      if (dlist && dlist.length) {
-        pkgList = pkgList.concat(dlist);
-      }
-      const pkgMap = getPipFrozenTree(basePath, f, tempDir);
-      if (pkgMap.pkgList && pkgMap.pkgList.length) {
-        pkgList = pkgList.concat(pkgMap.pkgList);
+      let retMap = await parsePoetrylockData(lockData, f);
+      if (retMap.pkgList && retMap.pkgList.length) {
+        pkgList = pkgList.concat(retMap.pkgList);
+        pkgList = trimComponents(pkgList, "json");
       }
-      if (pkgMap.dependenciesList) {
+      if (retMap.dependenciesList && retMap.dependenciesList.length) {
         dependencies = mergeDependencies(
           dependencies,
-          pkgMap.dependenciesList,
+          retMap.dependenciesList,
           parentComponent
         );
       }
+      // Retrieve the tree using virtualenv in deep mode and as a fallback
+      // This is a slow operation
+      if (options.deep || !dependencies.length) {
+        retMap = getPipFrozenTree(basePath, f, tempDir);
+        if (retMap.pkgList && retMap.pkgList.length) {
+          pkgList = pkgList.concat(retMap.pkgList);
+        }
+        if (retMap.dependenciesList) {
+          dependencies = mergeDependencies(
+            dependencies,
+            retMap.dependenciesList,
+            parentComponent
+          );
+        }
+      }
       const parentDependsOn = [];
       // Complete the dependency tree by making parent component depend on the first level
-      for (const p of pkgMap.rootList) {
+      for (const p of retMap.rootList) {
         parentDependsOn.push(`pkg:pypi/${p.name}@${p.version}`);
       }
       const pdependencies = {
@@ -2396,6 +2410,8 @@ export const createPythonBom = async (path, options) => {
           let frozen = false;
           // Attempt to pip freeze in a virtualenv to improve precision
           if (options.installDeps) {
+            // If there are multiple requirements files then the tree is getting constructed for each one
+            // adding to the delay.
             const pkgMap = getPipFrozenTree(basePath, f, tempDir);
             if (pkgMap.pkgList && pkgMap.pkgList.length) {
               pkgList = pkgList.concat(pkgMap.pkgList);
@@ -2453,7 +2469,7 @@ export const createPythonBom = async (path, options) => {
       }
       // Get the imported modules and a dedupe list of packages
       const parentDependsOn = new Set();
-      const retMap = await getPyModules(path, pkgList);
+      const retMap = await getPyModules(path, pkgList, options);
       if (retMap.pkgList && retMap.pkgList.length) {
         pkgList = pkgList.concat(retMap.pkgList);
         for (const p of retMap.pkgList) {
@@ -2547,6 +2563,9 @@ export const createPythonBom = async (path, options) => {
  */
 export const createGoBom = async (path, options) => {
   let pkgList = [];
+  let dependencies = [];
+  const allImports = {};
+  let parentComponent = createDefaultParentComponent(path, "golang", options);
   // Is this a binary file
   let maybeBinary = false;
   try {
@@ -2568,6 +2587,8 @@ export const createGoBom = async (path, options) => {
     }
     return buildBomNSData(options, pkgList, "golang", {
       allImports,
+      dependencies,
+      parentComponent,
       src: path,
       filename: path
     });
@@ -2597,8 +2618,70 @@ export const createGoBom = async (path, options) => {
         pkgList = pkgList.concat(dlist);
       }
     }
+    const doneList = {};
+    let circuitBreak = false;
+    if (DEBUG_MODE) {
+      console.log(
+        `Attempting to detect required packages using "go mod why" command for ${pkgList.length} packages`
+      );
+    }
+    // Using go mod why detect required packages
+    for (const apkg of pkgList) {
+      if (circuitBreak) {
+        break;
+      }
+      const pkgFullName = `${apkg.name}`;
+      if (apkg.scope === "required") {
+        allImports[pkgFullName] = true;
+        continue;
+      }
+      if (
+        apkg.scope === "optional" ||
+        allImports[pkgFullName] ||
+        doneList[pkgFullName]
+      ) {
+        continue;
+      }
+      if (DEBUG_MODE) {
+        console.log(`go mod why -m -vendor ${pkgFullName}`);
+      }
+      const mresult = spawnSync(
+        "go",
+        ["mod", "why", "-m", "-vendor", pkgFullName],
+        { cwd: path, encoding: "utf-8", timeout: TIMEOUT_MS }
+      );
+      if (mresult.status !== 0 || mresult.error) {
+        if (DEBUG_MODE) {
+          if (mresult.stdout) {
+            console.log(mresult.stdout);
+          }
+          if (mresult.stderr) {
+            console.log(mresult.stderr);
+          }
+        }
+        circuitBreak = true;
+      } else {
+        const mstdout = mresult.stdout;
+        if (mstdout) {
+          const cmdOutput = Buffer.from(mstdout).toString();
+          const whyPkg = parseGoModWhy(cmdOutput);
+          // whyPkg would include this package string
+          // github.com/golang/protobuf/proto github.com/golang/protobuf
+          // golang.org/x/tools/cmd/goimports golang.org/x/tools
+          if (whyPkg && whyPkg.includes(pkgFullName)) {
+            allImports[pkgFullName] = true;
+          }
+          doneList[pkgFullName] = true;
+        }
+      }
+    }
+    if (DEBUG_MODE) {
+      console.log(`Required packages: ${Object.keys(allImports).length}`);
+    }
     return buildBomNSData(options, pkgList, "golang", {
       src: path,
+      dependencies,
+      parentComponent,
       filename: gosumFiles.join(", ")
     });
   }
@@ -2614,7 +2697,7 @@ export const createGoBom = async (path, options) => {
       const dlist = await parseGosumData(gosumData);
       if (dlist && dlist.length) {
         dlist.forEach((pkg) => {
-          gosumMap[`${pkg.group}/${pkg.name}/${pkg.version}`] = pkg._integrity;
+          gosumMap[`${pkg.group}/${pkg.name}@${pkg.version}`] = pkg._integrity;
         });
       }
     }
@@ -2633,7 +2716,7 @@ export const createGoBom = async (path, options) => {
   );
   if (gomodFiles.length) {
     let shouldManuallyParse = false;
-    // Use the go list -deps and go mod why commands to generate a good quality BoM for non-docker invocations
+    // Use the go list -deps and go mod why commands to generate a good quality BOM for non-docker invocations
     if (!["docker", "oci", "os"].includes(options.projectType)) {
       for (const f of gomodFiles) {
         const basePath = dirname(f);
@@ -2641,36 +2724,85 @@ export const createGoBom = async (path, options) => {
         if (basePath.includes("/vendor/") || basePath.includes("/build/")) {
           continue;
         }
+        // First we execute the go list -deps command which gives the correct list of dependencies
         if (DEBUG_MODE) {
           console.log("Executing go list -deps in", basePath);
         }
-        const result = spawnSync(
+        let result = spawnSync(
           "go",
           [
             "list",
             "-deps",
             "-f",
-            "'{{with .Module}}{{.Path}} {{.Version}} {{.Indirect}} {{.GoMod}} {{.GoVersion}}{{end}}'",
+            "'{{with .Module}}{{.Path}} {{.Version}} {{.Indirect}} {{.GoMod}} {{.GoVersion}} {{.Main}}{{end}}'",
             "./..."
           ],
           { cwd: basePath, encoding: "utf-8", timeout: TIMEOUT_MS }
         );
+        if (DEBUG_MODE) {
+          console.log("Executing go mod graph in", basePath);
+        }
         if (result.status !== 0 || result.error) {
           shouldManuallyParse = true;
-          if (result.stdout) {
+          if (DEBUG_MODE && result.stdout) {
             console.log(result.stdout);
           }
-          if (result.stderr) {
+          if (DEBUG_MODE && result.stderr) {
             console.log(result.stderr);
           }
           options.failOnError && process.exit(1);
         }
         const stdout = result.stdout;
         if (stdout) {
-          const cmdOutput = Buffer.from(stdout).toString();
-          const dlist = await parseGoListDep(cmdOutput, gosumMap);
-          if (dlist && dlist.length) {
-            pkgList = pkgList.concat(dlist);
+          let cmdOutput = Buffer.from(stdout).toString();
+          const retMap = await parseGoListDep(cmdOutput, gosumMap);
+          if (retMap.pkgList && retMap.pkgList.length) {
+            pkgList = pkgList.concat(retMap.pkgList);
+          }
+          // We treat the main module as our parent
+          if (
+            retMap.parentComponent &&
+            Object.keys(retMap.parentComponent).length
+          ) {
+            parentComponent = retMap.parentComponent;
+            parentComponent.type = "application";
+          }
+          // Next we use the go mod graph command to construct the dependency tree
+          result = spawnSync("go", ["mod", "graph"], {
+            cwd: basePath,
+            encoding: "utf-8",
+            timeout: TIMEOUT_MS
+          });
+          // Check if got a mod graph successfully
+          if (result.status !== 0 || result.error) {
+            if (DEBUG_MODE && result.stdout) {
+              console.log(result.stdout);
+            }
+            if (DEBUG_MODE && result.stderr) {
+              console.log(result.stderr);
+            }
+            options.failOnError && process.exit(1);
+          }
+          if (result.stdout) {
+            cmdOutput = Buffer.from(result.stdout).toString();
+            const retMap = await parseGoModGraph(
+              cmdOutput,
+              f,
+              gosumMap,
+              pkgList,
+              parentComponent
+            );
+            if (retMap.pkgList && retMap.pkgList.length) {
+              pkgList = pkgList.concat(retMap.pkgList);
+              pkgList = trimComponents(pkgList, "json");
+            }
+            if (retMap.dependenciesList && retMap.dependenciesList.length) {
+              dependencies = mergeDependencies(
+                dependencies,
+                retMap.dependenciesList,
+                parentComponent
+              );
+            }
           }
         } else {
           shouldManuallyParse = true;
@@ -2680,67 +2812,20 @@ export const createGoBom = async (path, options) => {
           options.failOnError && process.exit(1);
         }
       }
-      const allImports = {};
-      let circuitBreak = false;
-      if (DEBUG_MODE) {
-        console.log(
-          `Attempting to detect required packages using "go mod why" command for ${pkgList.length} packages`
-        );
-      }
-      // Using go mod why detect required packages
-      for (const apkg of pkgList) {
-        if (circuitBreak) {
-          break;
-        }
-        const pkgFullName = `${apkg.name}`;
-        if (apkg.scope === "required") {
-          allImports[pkgFullName] = true;
-          continue;
-        }
-        if (DEBUG_MODE) {
-          console.log(`go mod why -m -vendor ${pkgFullName}`);
-        }
-        const mresult = spawnSync(
-          "go",
-          ["mod", "why", "-m", "-vendor", pkgFullName],
-          { cwd: path, encoding: "utf-8", timeout: TIMEOUT_MS }
-        );
-        if (mresult.status !== 0 || mresult.error) {
-          if (DEBUG_MODE) {
-            if (mresult.stdout) {
-              console.log(mresult.stdout);
-            }
-            if (mresult.stderr) {
-              console.log(mresult.stderr);
-            }
-          }
-          circuitBreak = true;
-        } else {
-          const mstdout = mresult.stdout;
-          if (mstdout) {
-            const cmdOutput = Buffer.from(mstdout).toString();
-            const whyPkg = parseGoModWhy(cmdOutput);
-            if (whyPkg == pkgFullName) {
-              allImports[pkgFullName] = true;
-            }
-          }
-        }
-      }
-      if (DEBUG_MODE) {
-        console.log(`Required packages: ${Object.keys(allImports).length}`);
-      }
       if (pkgList.length && !shouldManuallyParse) {
         return buildBomNSData(options, pkgList, "golang", {
           allImports,
+          dependencies,
+          parentComponent,
           src: path,
           filename: gomodFiles.join(", ")
         });
       }
     }
-    // Parse the gomod files manually. The resultant BoM would be incomplete
+    // Parse the gomod files manually. The resultant BOM would be incomplete
     if (!["docker", "oci", "os"].includes(options.projectType)) {
       console.log(
-        "Manually parsing go.mod files. The resultant BoM would be incomplete."
+        "Manually parsing go.mod files. The resultant BOM would be incomplete."
       );
     }
     for (const f of gomodFiles) {
@@ -2755,6 +2840,8 @@ export const createGoBom = async (path, options) => {
     }
     return buildBomNSData(options, pkgList, "golang", {
       src: path,
+      dependencies,
+      parentComponent,
       filename: gomodFiles.join(", ")
     });
   } else if (gopkgLockFiles.length) {
@@ -2772,6 +2859,8 @@ export const createGoBom = async (path, options) => {
     }
     return buildBomNSData(options, pkgList, "golang", {
       src: path,
+      dependencies,
+      parentComponent,
       filename: gopkgLockFiles.join(", ")
     });
   }
@@ -3284,7 +3373,7 @@ export const createCloudBuildBom = (path, options) => {
  */
 export const createOSBom = (path, options) => {
   console.warn(
-    "About to generate OBoM for the current OS installation. This will take several minutes ..."
+    "About to generate OBOM for the current OS installation. This will take several minutes ..."
   );
   let pkgList = [];
   let bomData = {};
@@ -3963,6 +4052,10 @@ export const createCsharpBom = async (
     path,
     (options.multiProject ? "**/" : "") + "packages.lock.json"
   );
+  const paketLockFiles = getAllFiles(
+    path,
+    (options.multiProject ? "**/" : "") + "paket.lock"
+  );
   const nupkgFiles = getAllFiles(
     path,
     (options.multiProject ? "**/" : "") + "*.nupkg"
@@ -4046,6 +4139,20 @@ export const createCsharpBom = async (
       }
     }
   }
+  if (paketLockFiles.length) {
+    manifestFiles = manifestFiles.concat(paketLockFiles);
+    // paket.lock parsing
+    for (const f of paketLockFiles) {
+      if (DEBUG_MODE) {
+        console.log(`Parsing ${f}`);
+      }
+      pkgData = readFileSync(f, { encoding: "utf-8" });
+      const dlist = await parsePaketLockData(pkgData);
+      if (dlist && dlist.length) {
+        pkgList = pkgList.concat(dlist);
+      }
+    }
+  }
   if (!parentComponent) {
     parentComponent = createDefaultParentComponent(path, options.type, options);
   }
@@ -4110,13 +4217,16 @@ export const trimComponents = (components, format) => {
   const filteredComponents = [];
   for (const comp of components) {
     if (format === "xml" && comp.component) {
-      const key = comp.component.purl || comp.component["bom-ref"];
+      const key =
+        comp.component.purl ||
+        comp.component["bom-ref"] ||
+        comp.name + comp.version;
       if (!keyCache[key]) {
         keyCache[key] = true;
         filteredComponents.push(comp);
       }
     } else {
-      const key = comp.purl || comp["bom-ref"];
+      const key = comp.purl || comp["bom-ref"] || comp.name + comp.version;
       if (!keyCache[key]) {
         keyCache[key] = true;
         filteredComponents.push(comp);
@@ -4143,7 +4253,7 @@ export const dedupeBom = (
   componentsXmls = trimComponents(componentsXmls, "xml");
   if (DEBUG_MODE) {
     console.log(
-      `BoM includes ${components.length} components and ${dependencies.length} dependencies after dedupe`
+      `BOM includes ${components.length} components and ${dependencies.length} dependencies after dedupe`
     );
   }
   const serialNum = "urn:uuid:" + uuidv4();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cyclonedx/cdxgen",
-  "version": "9.8.6",
+  "version": "9.8.8",
   "description": "Creates CycloneDX Software Bill of Materials (SBOM) from source or container image",
   "homepage": "http://github.com/cyclonedx/cdxgen",
   "author": "Prabhu Subramanian <prabhu@appthreat.com>",
@@ -31,11 +31,11 @@
   "type": "module",
   "exports": "./index.js",
   "bin": {
-    "cdxgen": "./bin/cdxgen.js",
-    "obom": "./bin/cdxgen.js",
-    "cdxi": "./bin/repl.js",
-    "evinse": "./bin/evinse.js",
-    "cdx-verify": "./bin/verify.js"
+    "cdxgen": "bin/cdxgen.js",
+    "obom": "bin/cdxgen.js",
+    "cdxi": "bin/repl.js",
+    "evinse": "bin/evinse.js",
+    "cdx-verify": "bin/verify.js"
   },
   "scripts": {
     "docs": "docsify serve docs",
@@ -49,7 +49,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/CycloneDX/cdxgen"
+    "url": "git+https://github.com/CycloneDX/cdxgen.git"
   },
   "bugs": {
     "url": "https://github.com/cyclonedx/cdxgen/issues"
@@ -82,7 +82,7 @@
     "yargs": "^17.7.2"
   },
   "optionalDependencies": {
-    "@appthreat/atom": "^1.2.3",
+    "@appthreat/atom": "1.2.5",
     "@cyclonedx/cdxgen-plugins-bin": "^1.4.0",
     "@cyclonedx/cdxgen-plugins-bin-arm64": "^1.4.0",
     "@cyclonedx/cdxgen-plugins-bin-ppc64": "^1.4.0",
@@ -107,4 +107,4 @@
     "jest": "^29.7.0",
     "prettier": "3.0.3"
   }
-}
+}

package/server.js

@@ -1,7 +1,7 @@
 import connect from "connect";
 import http from "node:http";
 import bodyParser from "body-parser";
-import url from "node:url";
+import url, { URL } from "node:url";
 import { spawnSync } from "node:child_process";
 import os from "node:os";
 import fs from "node:fs";
@@ -24,10 +24,14 @@ app.use(
 app.use(compression());
 
 const gitClone = (repoUrl) => {
+  const parsedUrl = new URL(repoUrl);
+
+  const sanitizedRepoUrl = `${parsedUrl.protocol}//${parsedUrl.host}${parsedUrl.pathname}`;
+
   const tempDir = fs.mkdtempSync(
-    path.join(os.tmpdir(), path.basename(repoUrl))
+    path.join(os.tmpdir(), path.basename(parsedUrl.pathname))
   );
-  console.log("Cloning", repoUrl, "to", tempDir);
+  console.log("Cloning", sanitizedRepoUrl, "to", tempDir);
   const result = spawnSync("git", ["clone", repoUrl, "--depth", "1", tempDir], {
     encoding: "utf-8",
     shell: false