@cyclonedx/cdxgen 8.1.5 → 8.1.7

package/README.md

@@ -52,7 +52,7 @@ NOTE:
 Footnotes:
 
 - [1] - For multi-module application, the BoM file could include components that may not be included in the packaged war or ear file.
-- [2] - Use pip freeze to improve the accuracy for requirements.txt based parsing. `python -m pip freeze > requirements.txt`
+- [2] - Pip freeze is automatically performed to improve precision.
 - [3] - Perform dotnet or nuget restore to generate project.assets.json. Without this file cdxgen would not include indirect dependencies.
 - [4] - See section on plugins
 - [5] - Powered by osquery. See section on plugins
@@ -86,36 +86,38 @@ docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/cyclonedx/cdxgen -r /a
 ```text
 $ cdxgen -h
 Options:
-  -o, --output           Output file for bom.xml or bom.json. Default console
-  -t, --type             Project type
-  -r, --recurse          Recurse mode suitable for mono-repos          [boolean]
-  -p, --print            Print the SBoM as a table                     [boolean]
-  -c, --resolve-class    Resolve class names for packages. jars only for now.
-                                                                       [boolean]
-      --deep             Perform deep searches for components. Useful while
-                         scanning live OS and oci images.              [boolean]
-      --server-url       Dependency track url. Eg:
-                         https://deptrack.cyclonedx.io
-      --api-key          Dependency track api key
-      --project-group    Dependency track project group
-      --project-name     Dependency track project name. Default use
-                         the directory name
-      --project-version  Dependency track project version
-      --project-id       Dependency track project id. Either
-                         provide the id or the project name and version together
-      --required-only    Include only the packages with required scope on the
-                         SBoM.                                         [boolean]
-      --fail-on-error    Fail if any dependency extractor fails.       [boolean]
-      --no-babel         Do not use babel to perform usage analysis for
-                         JavaScript/TypeScript projects.               [boolean]
+  -o, --output                 Output file for bom.xml or bom.json. Default
+                               bom.json
+  -t, --type                   Project type
+  -r, --recurse                Recurse mode suitable for mono-repos    [boolean]
+  -p, --print                  Print the SBoM as a table. Defaults to true if
+                               output file is not specified with -o    [boolean]
+  -c, --resolve-class          Resolve class names for packages. jars only for
+                               now.                                    [boolean]
+      --deep                   Perform deep searches for components. Useful
+                               while scanning live OS and oci images.  [boolean]
+      --server-url             Dependency track url. Eg:
+                               https://deptrack.cyclonedx.io
+      --api-key                Dependency track api key
+      --project-group          Dependency track project group
+      --project-name           Dependency track project name. Default use the
+                               directory name
+      --project-version        Dependency track project version    [default: ""]
+      --project-id             Dependency track project id. Either provide the
+                               id or the project name and version together
+      --required-only          Include only the packages with required scope on
+                               the SBoM.                               [boolean]
+      --fail-on-error          Fail if any dependency extractor fails. [boolean]
+      --no-babel               Do not use babel to perform usage analysis for
+                               JavaScript/TypeScript projects.         [boolean]
       --generate-key-and-sign  Generate an RSA public/private key pair and then
                                sign the generated SBoM using JSON Web
                                Signatures.                             [boolean]
       --server                 Run cdxgen as a server                  [boolean]
       --server-host            Listen address             [default: "127.0.0.1"]
       --server-port            Listen port                     [default: "9090"]
-      --version          Show version number                           [boolean]
-  -h                     Show help                                     [boolean]
+      --version                Show version number                     [boolean]
+  -h                           Show help                               [boolean]
 ```
 
 ## Example

package/analyzer.js

@@ -13,10 +13,19 @@ const IGNORE_DIRS = [
   "e2e",
   "examples",
   "cypress",
-  "site-packages"
+  "site-packages",
+  "typings",
+  "api_docs",
+  "dev_docs",
+  "types",
+  "mock",
+  "mocks"
 ];
 
-const IGNORE_FILE_PATTERN = new RegExp("(conf|test|spec|mock)\\.(js|ts)$", "i");
+const IGNORE_FILE_PATTERN = new RegExp(
+  "(conf|test|spec|mock|\\.d)\\.(js|ts|tsx)$",
+  "i"
+);
 
 const getAllFiles = (dir, extn, files, result, regex) => {
   files = files || fs.readdirSync(dir);

package/bin/cdxgen

@@ -10,7 +10,7 @@ const bomServer = require("../server.js");
 const args = require("yargs")
   .option("output", {
     alias: "o",
-    description: "Output file for bom.xml or bom.json. Default console"
+    description: "Output file for bom.xml or bom.json. Default bom.json"
   })
   .option("type", {
     alias: "t",
@@ -24,7 +24,8 @@ const args = require("yargs")
   .option("print", {
     alias: "p",
     type: "boolean",
-    description: "Print the SBoM as a table"
+    description:
+      "Print the SBoM as a table. Defaults to true if output file is not specified with -o"
   })
   .option("resolve-class", {
     alias: "c",
@@ -154,7 +155,10 @@ let options = {
     return await bomServer.start(options);
   }
   const bomNSData = (await bom.createBom(filePath, options)) || {};
-
+  if (!args.output) {
+    args.output = "bom.json";
+    args.print = true;
+  }
   if (
     args.output &&
     (typeof args.output === "string" || args.output instanceof String)
@@ -286,8 +290,9 @@ let options = {
   }
 
   // Automatically submit the bom data
-  if (args.serverUrl && args.apiKey) {
+  if (args.serverUrl && args.serverUrl != true && args.apiKey) {
     try {
+      // TODO: Need to use json format for v9
       const dbody = await bom.submitBom(args, bomNSData.bomXml);
       console.log("Response from server", dbody);
     } catch (err) {

package/index.js

@@ -39,6 +39,11 @@ if (process.env.LEIN_CMD) {
   LEIN_CMD = process.env.LEIN_CMD;
 }
 
+let PIP_CMD = "pip";
+if (process.env.PIP_CMD) {
+  PIP_CMD = process.env.PIP_CMD;
+}
+
 // Construct sbt cache directory
 let SBT_CACHE_DIR =
   process.env.SBT_CACHE_DIR || pathLib.join(os.homedir(), ".ivy2", "cache");
@@ -1887,7 +1892,7 @@ const createPythonBom = async (path, options) => {
   );
   const reqFiles = utils.getAllFiles(
     path,
-    (options.multiProject ? "**/" : "") + "requirements.txt"
+    (options.multiProject ? "**/" : "") + "*requirements.txt"
   );
   const reqDirFiles = utils.getAllFiles(
     path,
@@ -1977,7 +1982,28 @@ const createPythonBom = async (path, options) => {
       metadataFilename = "requirements.txt";
       if (reqFiles && reqFiles.length) {
         for (let f of reqFiles) {
-          const reqData = fs.readFileSync(f, { encoding: "utf-8" });
+          const basePath = pathLib.dirname(f);
+          let reqData = undefined;
+          // Attempt to pip freeze to improve precision
+          if (options.installDeps) {
+            const result = spawnSync(PIP_CMD, ["freeze", "-r", f, "-l"], {
+              cwd: basePath,
+              encoding: "utf-8",
+              timeout: TIMEOUT_MS
+            });
+            if (result.status === 0 && result.stdout) {
+              reqData = Buffer.from(result.stdout).toString();
+            }
+          }
+          // Fallback to parsing requirements file
+          if (!reqData) {
+            if (DEBUG_MODE) {
+              console.log(
+                `Falling back to manually parsing ${f}. The result would be incomplete!`
+              );
+            }
+            reqData = fs.readFileSync(f, { encoding: "utf-8" });
+          }
           const dlist = await utils.parseReqFile(reqData);
           if (dlist && dlist.length) {
             pkgList = pkgList.concat(dlist);
@@ -3817,7 +3843,7 @@ const createXBom = async (path, options) => {
   const poetryMode = fs.existsSync(pathLib.join(path, "poetry.lock"));
   const reqFiles = utils.getAllFiles(
     path,
-    (options.multiProject ? "**/" : "") + "requirements.txt"
+    (options.multiProject ? "**/" : "") + "*requirements.txt"
   );
   const reqDirFiles = utils.getAllFiles(
     path,
@@ -4280,15 +4306,25 @@ exports.submitBom = async (args, bomContents) => {
   if (encodedBomContents.startsWith("77u/")) {
     encodedBomContents = encodedBomContents.substring(4);
   }
+  let projectVersion = args.projectVersion || "master";
+  if (projectVersion == true) {
+    projectVersion = "master";
+  }
   const bomPayload = {
     project: args.projectId,
     projectName: args.projectName,
-    projectVersion: args.projectVersion,
+    projectVersion: projectVersion,
     autoCreate: "true",
     bom: encodedBomContents
   };
   if (DEBUG_MODE) {
-    console.log("Submitting BOM to", serverUrl);
+    console.log(
+      "Submitting BOM to",
+      serverUrl,
+      "params",
+      args.projectName,
+      projectVersion
+    );
   }
   return await got(serverUrl, {
     method: "PUT",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cyclonedx/cdxgen",
-  "version": "8.1.5",
+  "version": "8.1.7",
   "description": "Creates CycloneDX Software Bill-of-Materials (SBOM) from source or container image",
   "homepage": "http://github.com/cyclonedx/cdxgen",
   "author": "Prabhu Subramanian <prabhu@appthreat.com>",

package/utils.js

@@ -1756,7 +1756,13 @@ exports.parsePoetrylockData = parsePoetrylockData;
 const parseReqFile = async function (reqData) {
   const pkgList = [];
   let fetchIndirectDeps = false;
+  let compScope = undefined;
   reqData.split("\n").forEach((l) => {
+    if (l.includes("# Basic requirements")) {
+      compScope = "required";
+    } else if (l.includes("added by pip freeze")) {
+      compScope = undefined;
+    }
     if (!l.startsWith("#")) {
       if (l.indexOf("=") > -1) {
         let tmpA = l.split(/(==|<=|~=|>=)/);
@@ -1773,7 +1779,8 @@ const parseReqFile = async function (reqData) {
         if (!tmpA[0].includes("=") && !tmpA[0].trim().includes(" ")) {
           pkgList.push({
             name: tmpA[0].trim(),
-            version: versionStr
+            version: versionStr,
+            scope: compScope
           });
         }
       } else if (/[>|[|@]/.test(l)) {
@@ -1784,7 +1791,8 @@ const parseReqFile = async function (reqData) {
         if (!tmpA[0].trim().includes(" ")) {
           pkgList.push({
             name: tmpA[0].trim(),
-            version: null
+            version: null,
+            scope: compScope
           });
         }
       } else if (l) {
@@ -1795,7 +1803,8 @@ const parseReqFile = async function (reqData) {
         if (!l.includes(" ")) {
           pkgList.push({
             name: l,
-            version: null
+            version: null,
+            scope: compScope
           });
         }
       }

package/utils.test.js

@@ -1348,7 +1348,7 @@ test("parseGemspecData", async () => {
   });
 });
 
-test("parse requirements.txt with comments", async () => {
+test("parse requirements.txt", async () => {
   jest.setTimeout(120000);
   let deps = await utils.parseReqFile(
     fs.readFileSync(
@@ -1357,6 +1357,15 @@ test("parse requirements.txt with comments", async () => {
     )
   );
   expect(deps.length).toEqual(31);
+  deps = await utils.parseReqFile(
+    fs.readFileSync("./test/data/requirements.freeze.txt", (encoding = "utf-8"))
+  );
+  expect(deps.length).toEqual(113);
+  expect(deps[0]).toEqual({
+    name: "elasticsearch",
+    version: "8.6.2",
+    scope: "required"
+  });
 });
 
 test("parse poetry.lock", async () => {