@cyclonedx/cdxgen 8.1.4 → 8.1.6

package/README.md

@@ -52,7 +52,7 @@ NOTE:
 Footnotes:
 
 - [1] - For multi-module application, the BoM file could include components that may not be included in the packaged war or ear file.
-- [2] - Use pip freeze to improve the accuracy for requirements.txt based parsing. `python -m pip freeze > requirements.txt`
+- [2] - Pip freeze is automatically performed to improve precision.
 - [3] - Perform dotnet or nuget restore to generate project.assets.json. Without this file cdxgen would not include indirect dependencies.
 - [4] - See section on plugins
 - [5] - Powered by osquery. See section on plugins
@@ -86,36 +86,38 @@ docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/cyclonedx/cdxgen -r /a
 ```text
 $ cdxgen -h
 Options:
-  -o, --output           Output file for bom.xml or bom.json. Default console
-  -t, --type             Project type
-  -r, --recurse          Recurse mode suitable for mono-repos          [boolean]
-  -p, --print            Print the SBoM as a table                     [boolean]
-  -c, --resolve-class    Resolve class names for packages. jars only for now.
-                                                                       [boolean]
-      --deep             Perform deep searches for components. Useful while
-                         scanning live OS and oci images.              [boolean]
-      --server-url       Dependency track url. Eg:
-                         https://deptrack.cyclonedx.io
-      --api-key          Dependency track api key
-      --project-group    Dependency track project group
-      --project-name     Dependency track project name. Default use
-                         the directory name
-      --project-version  Dependency track project version
-      --project-id       Dependency track project id. Either
-                         provide the id or the project name and version together
-      --required-only    Include only the packages with required scope on the
-                         SBoM.                                         [boolean]
-      --fail-on-error    Fail if any dependency extractor fails.       [boolean]
-      --no-babel         Do not use babel to perform usage analysis for
-                         JavaScript/TypeScript projects.               [boolean]
+  -o, --output                 Output file for bom.xml or bom.json. Default
+                               bom.json
+  -t, --type                   Project type
+  -r, --recurse                Recurse mode suitable for mono-repos    [boolean]
+  -p, --print                  Print the SBoM as a table. Defaults to true if
+                               output file is not specified with -o    [boolean]
+  -c, --resolve-class          Resolve class names for packages. jars only for
+                               now.                                    [boolean]
+      --deep                   Perform deep searches for components. Useful
+                               while scanning live OS and oci images.  [boolean]
+      --server-url             Dependency track url. Eg:
+                               https://deptrack.cyclonedx.io
+      --api-key                Dependency track api key
+      --project-group          Dependency track project group
+      --project-name           Dependency track project name. Default use the
+                               directory name
+      --project-version        Dependency track project version    [default: ""]
+      --project-id             Dependency track project id. Either provide the
+                               id or the project name and version together
+      --required-only          Include only the packages with required scope on
+                               the SBoM.                               [boolean]
+      --fail-on-error          Fail if any dependency extractor fails. [boolean]
+      --no-babel               Do not use babel to perform usage analysis for
+                               JavaScript/TypeScript projects.         [boolean]
       --generate-key-and-sign  Generate an RSA public/private key pair and then
                                sign the generated SBoM using JSON Web
                                Signatures.                             [boolean]
       --server                 Run cdxgen as a server                  [boolean]
       --server-host            Listen address             [default: "127.0.0.1"]
       --server-port            Listen port                     [default: "9090"]
-      --version          Show version number                           [boolean]
-  -h                     Show help                                     [boolean]
+      --version                Show version number                     [boolean]
+  -h                           Show help                               [boolean]
 ```
 
 ## Example

package/analyzer.js

@@ -13,10 +13,19 @@ const IGNORE_DIRS = [
   "e2e",
   "examples",
   "cypress",
-  "site-packages"
+  "site-packages",
+  "typings",
+  "api_docs",
+  "dev_docs",
+  "types",
+  "mock",
+  "mocks"
 ];
 
-const IGNORE_FILE_PATTERN = new RegExp("(conf|test|spec|mock)\\.(js|ts)$", "i");
+const IGNORE_FILE_PATTERN = new RegExp(
+  "(conf|test|spec|mock|\\.d)\\.(js|ts|tsx)$",
+  "i"
+);
 
 const getAllFiles = (dir, extn, files, result, regex) => {
   files = files || fs.readdirSync(dir);

package/bin/cdxgen

@@ -10,7 +10,7 @@ const bomServer = require("../server.js");
 const args = require("yargs")
   .option("output", {
     alias: "o",
-    description: "Output file for bom.xml or bom.json. Default console"
+    description: "Output file for bom.xml or bom.json. Default bom.json"
   })
   .option("type", {
     alias: "t",
@@ -24,7 +24,8 @@ const args = require("yargs")
   .option("print", {
     alias: "p",
     type: "boolean",
-    description: "Print the SBoM as a table"
+    description:
+      "Print the SBoM as a table. Defaults to true if output file is not specified with -o"
   })
   .option("resolve-class", {
     alias: "c",
@@ -154,7 +155,10 @@ let options = {
     return await bomServer.start(options);
   }
   const bomNSData = (await bom.createBom(filePath, options)) || {};
-
+  if (!args.output) {
+    args.output = "bom.json";
+    args.print = true;
+  }
   if (
     args.output &&
     (typeof args.output === "string" || args.output instanceof String)
@@ -286,8 +290,9 @@ let options = {
   }
 
   // Automatically submit the bom data
-  if (args.serverUrl && args.apiKey) {
+  if (args.serverUrl && args.serverUrl != true && args.apiKey) {
     try {
+      // TODO: Need to use json format for v9
       const dbody = await bom.submitBom(args, bomNSData.bomXml);
       console.log("Response from server", dbody);
     } catch (err) {

package/docker.js

@@ -381,7 +381,7 @@ const getImage = async (fullImageName) => {
       `Unable to pull ${fullImageName}. Check if the name is valid. Perform any authentication prior to invoking cdxgen.`
     );
     console.log(
-      `Trying manually pulling this image using docker pull ${fullImageName}`
+      `Trying to manually pulling this image using docker pull ${fullImageName}`
     );
   }
   return localData;

package/index.js

@@ -39,6 +39,11 @@ if (process.env.LEIN_CMD) {
   LEIN_CMD = process.env.LEIN_CMD;
 }
 
+let PIP_CMD = "pip";
+if (process.env.PIP_CMD) {
+  PIP_CMD = process.env.PIP_CMD;
+}
+
 // Construct sbt cache directory
 let SBT_CACHE_DIR =
   process.env.SBT_CACHE_DIR || pathLib.join(os.homedir(), ".ivy2", "cache");
@@ -1887,7 +1892,7 @@ const createPythonBom = async (path, options) => {
   );
   const reqFiles = utils.getAllFiles(
     path,
-    (options.multiProject ? "**/" : "") + "requirements.txt"
+    (options.multiProject ? "**/" : "") + "*requirements.txt"
   );
   const reqDirFiles = utils.getAllFiles(
     path,
@@ -1977,7 +1982,28 @@ const createPythonBom = async (path, options) => {
       metadataFilename = "requirements.txt";
       if (reqFiles && reqFiles.length) {
         for (let f of reqFiles) {
-          const reqData = fs.readFileSync(f, { encoding: "utf-8" });
+          const basePath = pathLib.dirname(f);
+          let reqData = undefined;
+          // Attempt to pip freeze to improve precision
+          if (options.installDeps) {
+            const result = spawnSync(PIP_CMD, ["freeze", "-r", f, "-l"], {
+              cwd: basePath,
+              encoding: "utf-8",
+              timeout: TIMEOUT_MS
+            });
+            if (result.status === 0 && result.stdout) {
+              reqData = Buffer.from(result.stdout).toString();
+            }
+          }
+          // Fallback to parsing requirements file
+          if (!reqData) {
+            if (DEBUG_MODE) {
+              console.log(
+                `Falling back to manually parsing ${f}. The result would be incomplete!`
+              );
+            }
+            reqData = fs.readFileSync(f, { encoding: "utf-8" });
+          }
           const dlist = await utils.parseReqFile(reqData);
           if (dlist && dlist.length) {
             pkgList = pkgList.concat(dlist);
@@ -2104,6 +2130,7 @@ const createGoBom = async (path, options) => {
     (options.multiProject ? "**/" : "") + "go.mod"
   );
   if (gomodFiles.length) {
+    let shouldManuallyParse = false;
     // Use the go list -deps and go mod why commands to generate a good quality BoM for non-docker invocations
     if (!["docker", "oci", "os"].includes(options.projectType)) {
       for (let f of gomodFiles) {
@@ -2127,6 +2154,7 @@ const createGoBom = async (path, options) => {
           { cwd: basePath, encoding: "utf-8", timeout: TIMEOUT_MS }
         );
         if (result.status !== 0 || result.error) {
+          shouldManuallyParse = true;
           console.error(result.stdout, result.stderr);
           options.failOnError && process.exit(1);
         }
@@ -2138,6 +2166,7 @@ const createGoBom = async (path, options) => {
             pkgList = pkgList.concat(dlist);
           }
         } else {
+          shouldManuallyParse = true;
           console.error("go unexpectedly didn't return any output");
           options.failOnError && process.exit(1);
         }
@@ -2182,11 +2211,13 @@ const createGoBom = async (path, options) => {
       if (DEBUG_MODE) {
         console.log(`Required packages: ${Object.keys(allImports).length}`);
       }
-      return buildBomNSData(options, pkgList, "golang", {
-        allImports,
-        src: path,
-        filename: gomodFiles.join(", ")
-      });
+      if (pkgList.length && !shouldManuallyParse) {
+        return buildBomNSData(options, pkgList, "golang", {
+          allImports,
+          src: path,
+          filename: gomodFiles.join(", ")
+        });
+      }
     }
     // Parse the gomod files manually. The resultant BoM would be incomplete
     if (!["docker", "oci", "os"].includes(options.projectType)) {
@@ -3455,7 +3486,7 @@ const createMultiXBom = async (pathList, options) => {
     if (bomData && bomData.bomJson && bomData.bomJson.components) {
       if (DEBUG_MODE) {
         console.log(
-          `Found ${bomData.bomJson.components.length} node.js packages at ${path}`
+          `Found ${bomData.bomJson.components.length} npm packages at ${path}`
         );
       }
       components = components.concat(bomData.bomJson.components);
@@ -3812,7 +3843,7 @@ const createXBom = async (path, options) => {
   const poetryMode = fs.existsSync(pathLib.join(path, "poetry.lock"));
   const reqFiles = utils.getAllFiles(
     path,
-    (options.multiProject ? "**/" : "") + "requirements.txt"
+    (options.multiProject ? "**/" : "") + "*requirements.txt"
   );
   const reqDirFiles = utils.getAllFiles(
     path,
@@ -4275,15 +4306,25 @@ exports.submitBom = async (args, bomContents) => {
   if (encodedBomContents.startsWith("77u/")) {
     encodedBomContents = encodedBomContents.substring(4);
   }
+  const projectVersion =
+    args.projectVersion && args.projectVersion.length
+      ? args.projectVersion
+      : "master";
   const bomPayload = {
     project: args.projectId,
     projectName: args.projectName,
-    projectVersion: args.projectVersion,
+    projectVersion: projectVersion,
     autoCreate: "true",
     bom: encodedBomContents
   };
   if (DEBUG_MODE) {
-    console.log("Submitting BOM to", serverUrl);
+    console.log(
+      "Submitting BOM to",
+      serverUrl,
+      "params",
+      args.projectName,
+      projectVersion
+    );
   }
   return await got(serverUrl, {
     method: "PUT",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cyclonedx/cdxgen",
-  "version": "8.1.4",
+  "version": "8.1.6",
   "description": "Creates CycloneDX Software Bill-of-Materials (SBOM) from source or container image",
   "homepage": "http://github.com/cyclonedx/cdxgen",
   "author": "Prabhu Subramanian <prabhu@appthreat.com>",

package/utils.js

@@ -1756,7 +1756,13 @@ exports.parsePoetrylockData = parsePoetrylockData;
 const parseReqFile = async function (reqData) {
   const pkgList = [];
   let fetchIndirectDeps = false;
+  let compScope = undefined;
   reqData.split("\n").forEach((l) => {
+    if (l.includes("# Basic requirements")) {
+      compScope = "required";
+    } else if (l.includes("added by pip freeze")) {
+      compScope = undefined;
+    }
     if (!l.startsWith("#")) {
       if (l.indexOf("=") > -1) {
         let tmpA = l.split(/(==|<=|~=|>=)/);
@@ -1773,7 +1779,8 @@ const parseReqFile = async function (reqData) {
         if (!tmpA[0].includes("=") && !tmpA[0].trim().includes(" ")) {
           pkgList.push({
             name: tmpA[0].trim(),
-            version: versionStr
+            version: versionStr,
+            scope: compScope
           });
         }
       } else if (/[>|[|@]/.test(l)) {
@@ -1784,7 +1791,8 @@ const parseReqFile = async function (reqData) {
         if (!tmpA[0].trim().includes(" ")) {
           pkgList.push({
             name: tmpA[0].trim(),
-            version: null
+            version: null,
+            scope: compScope
           });
         }
       } else if (l) {
@@ -1795,7 +1803,8 @@ const parseReqFile = async function (reqData) {
         if (!l.includes(" ")) {
           pkgList.push({
             name: l,
-            version: null
+            version: null,
+            scope: compScope
           });
         }
       }
@@ -2799,10 +2808,11 @@ const recurseImageNameLookup = (keyValueObj, pkgList, imgList) => {
       typeof imageLike === "string" &&
       !imgList.includes(imageLike)
     ) {
-      if (imageLike.includes(":${VERSION:")) {
+      if (imageLike.includes("VERSION")) {
         imageLike = imageLike
           .replace(":${VERSION:-", ":")
           .replace(":${VERSION:", ":")
+          .replace(":%VERSION%", ":latest")
           .replace("}", "");
       }
       pkgList.push({ image: imageLike });

package/utils.test.js

@@ -1348,7 +1348,7 @@ test("parseGemspecData", async () => {
   });
 });
 
-test("parse requirements.txt with comments", async () => {
+test("parse requirements.txt", async () => {
   jest.setTimeout(120000);
   let deps = await utils.parseReqFile(
     fs.readFileSync(
@@ -1357,6 +1357,15 @@ test("parse requirements.txt with comments", async () => {
     )
   );
   expect(deps.length).toEqual(31);
+  deps = await utils.parseReqFile(
+    fs.readFileSync("./test/data/requirements.freeze.txt", (encoding = "utf-8"))
+  );
+  expect(deps.length).toEqual(113);
+  expect(deps[0]).toEqual({
+    name: "elasticsearch",
+    version: "8.6.2",
+    scope: "required"
+  });
 });
 
 test("parse poetry.lock", async () => {