@cyclonedx/cdxgen 8.5.2 → 8.6.0

package/data/python-stdlib.json

@@ -0,0 +1,307 @@
+[
+  "_abc",
+  "_ast",
+  "_codecs",
+  "_collections",
+  "_functools",
+  "_imp",
+  "_io",
+  "_locale",
+  "_operator",
+  "_peg_parser",
+  "_signal",
+  "_sre",
+  "_stat",
+  "_string",
+  "_symtable",
+  "_thread",
+  "_tracemalloc",
+  "_warnings",
+  "_weakref",
+  "atexit",
+  "basehttpserver",
+  "simplehttpserver",
+  "builtins",
+  "errno",
+  "faulthandler",
+  "gc",
+  "itertools",
+  "marshal",
+  "posix",
+  "pwd",
+  "sys",
+  "time",
+  "xxsubtype",
+  "__future__",
+  "_aix_support",
+  "_bootlocale",
+  "_bootsubprocess",
+  "_collections_abc",
+  "_compat_pickle",
+  "_compression",
+  "_markupbase",
+  "_osx_support",
+  "_py_abc",
+  "_pydecimal",
+  "_pyio",
+  "_sitebuiltins",
+  "_strptime",
+  "_sysconfigdata__linux_x86_64-linux-gnu",
+  "_sysconfigdata_d_linux_x86_64-linux-gnu",
+  "_threading_local",
+  "_weakrefset",
+  "abc",
+  "aifc",
+  "antigravity",
+  "argparse",
+  "ast",
+  "asynchat",
+  "asyncio",
+  "asyncore",
+  "base64",
+  "bdb",
+  "binhex",
+  "bisect",
+  "bz2",
+  "cprofile",
+  "calendar",
+  "cgi",
+  "cgitb",
+  "chunk",
+  "cmd",
+  "code",
+  "codecs",
+  "codeop",
+  "collections",
+  "colorsys",
+  "compileall",
+  "concurrent",
+  "configparser",
+  "contextlib",
+  "contextvars",
+  "copy",
+  "copyreg",
+  "crypt",
+  "csv",
+  "ctypes",
+  "curses",
+  "dataclasses",
+  "datetime",
+  "dbm",
+  "decimal",
+  "dummy_threading",
+  "difflib",
+  "dis",
+  "distutils",
+  "doctest",
+  "email",
+  "encodings",
+  "ensurepip",
+  "enum",
+  "filecmp",
+  "fileinput",
+  "fnmatch",
+  "formatter",
+  "fractions",
+  "ftplib",
+  "functools",
+  "genericpath",
+  "getopt",
+  "getpass",
+  "gettext",
+  "glob",
+  "graphlib",
+  "gzip",
+  "hashlib",
+  "heapq",
+  "hmac",
+  "html",
+  "http",
+  "imaplib",
+  "imghdr",
+  "imp",
+  "importlib",
+  "inspect",
+  "io",
+  "ipaddress",
+  "json",
+  "keyword",
+  "lib2to3",
+  "linecache",
+  "locale",
+  "logging",
+  "lzma",
+  "mailbox",
+  "mailcap",
+  "mimetypes",
+  "modulefinder",
+  "msvcrt",
+  "multiprocessing",
+  "netrc",
+  "nntplib",
+  "ntpath",
+  "nturl2path",
+  "numbers",
+  "openssl",
+  "opcode",
+  "operator",
+  "optparse",
+  "os",
+  "pathlib",
+  "pdb",
+  "pickle",
+  "pickletools",
+  "pipes",
+  "pkgutil",
+  "platform",
+  "plistlib",
+  "poplib",
+  "posixpath",
+  "pprint",
+  "profile",
+  "pstats",
+  "pty",
+  "py_compile",
+  "pyclbr",
+  "pydoc",
+  "pydoc_data",
+  "queue",
+  "quopri",
+  "random",
+  "re",
+  "reprlib",
+  "rlcompleter",
+  "runpy",
+  "sched",
+  "secrets",
+  "selectors",
+  "shelve",
+  "shlex",
+  "shutil",
+  "signal",
+  "site",
+  "smtpd",
+  "smtplib",
+  "sndhdr",
+  "socket",
+  "socketserver",
+  "sqlite3",
+  "sre_compile",
+  "sre_constants",
+  "sre_parse",
+  "ssl",
+  "stat",
+  "statistics",
+  "string",
+  "stringio",
+  "cstringio",
+  "stringprep",
+  "struct",
+  "subprocess",
+  "sunau",
+  "symbol",
+  "symtable",
+  "sysconfig",
+  "tabnanny",
+  "tarfile",
+  "telnetlib",
+  "tempfile",
+  "textwrap",
+  "this",
+  "threading",
+  "timeit",
+  "token",
+  "tokenize",
+  "trace",
+  "traceback",
+  "tracemalloc",
+  "tty",
+  "types",
+  "typing",
+  "unittest",
+  "urllib",
+  "uu",
+  "uuid",
+  "venv",
+  "warnings",
+  "wave",
+  "weakref",
+  "webbrowser",
+  "wsgiref",
+  "winreg",
+  "xdrlib",
+  "xml",
+  "xmlrpc",
+  "zipapp",
+  "zipfile",
+  "zipimport",
+  "zoneinfo",
+  "_asyncio",
+  "_bisect",
+  "_blake2",
+  "_bz2",
+  "_codecs_cn",
+  "_codecs_hk",
+  "_codecs_iso2022",
+  "_codecs_jp",
+  "_codecs_kr",
+  "_codecs_tw",
+  "_contextvars",
+  "_crypt",
+  "_csv",
+  "_ctypes",
+  "_curses",
+  "_curses_panel",
+  "_datetime",
+  "_dbm",
+  "_decimal",
+  "_elementtree",
+  "_gdbm",
+  "_hashlib",
+  "_heapq",
+  "_json",
+  "_lsprof",
+  "_lzma",
+  "_multibytecodec",
+  "_multiprocessing",
+  "_opcode",
+  "_pickle",
+  "_posixshmem",
+  "_posixsubprocess",
+  "_queue",
+  "_random",
+  "_socket",
+  "_sqlite3",
+  "_ssl",
+  "_statistics",
+  "_struct",
+  "_uuid",
+  "_xxsubinterpreters",
+  "_zoneinfo",
+  "array",
+  "audioop",
+  "binascii",
+  "cmath",
+  "fcntl",
+  "grp",
+  "math",
+  "mmap",
+  "ossaudiodev",
+  "parser",
+  "pyexpat",
+  "readline",
+  "resource",
+  "select",
+  "spwd",
+  "syslog",
+  "termios",
+  "unicodedata",
+  "xxlimited",
+  "zlib",
+  "_distutils_hack",
+  "pip",
+  "pkg_resources",
+  "setuptools",
+  "_info",
+  "_registry"
+]

package/{queries.json → data/queries.json}

@@ -46,23 +46,23 @@
     "purlType": "pypi"
   },
   "windows_programs": {
-    "query" : "select * from programs;",
-    "description" : "Retrieves the list of products as they are installed by Windows Installer in the target Windows system.",
+    "query": "select * from programs;",
+    "description": "Retrieves the list of products as they are installed by Windows Installer in the target Windows system.",
     "purlType": "swid"
   },
   "windows_patches": {
-    "query" : "select * from patches;",
-    "description" : "Retrieves all the information for the current windows drivers in the target Windows system.",
+    "query": "select * from patches;",
+    "description": "Retrieves all the information for the current windows drivers in the target Windows system.",
     "purlType": "swid"
   },
   "windows_drivers": {
-    "query" : "select * from drivers;",
-    "description" : "Retrieves all the information for the current windows drivers in the target Windows system.",
+    "query": "select * from drivers;",
+    "description": "Retrieves all the information for the current windows drivers in the target Windows system.",
     "purlType": "swid"
   },
   "windows_shared_resources": {
-    "query" : "select * from shared_resources;",
-    "description" : "Retrieves the list of shared resources in the target Windows system.",
+    "query": "select * from shared_resources;",
+    "description": "Retrieves the list of shared resources in the target Windows system.",
     "purlType": "swid"
   }
 }

package/data/vendor-alias.json

@@ -0,0 +1,10 @@
+{
+  "commons-": "org.apache.commons",
+  "spring-": "org.springframework",
+  "jackson-dataformat-": "com.fasterxml.jackson.dataformat",
+  "jackson-databind": "com.fasterxml.jackson.core",
+  "jackson-core": "com.fasterxml.jackson.core",
+  "jackson-annotations": "com.fasterxml.jackson.core",
+  "jackson-jaxrs-": "com.fasterxml.jackson.jaxrs",
+  "spring.boot": "org.springframework.boot"
+}

package/docker.js

@@ -224,6 +224,9 @@ const getConnection = async (options) => {
               "Ensure docker/podman service or Docker for Desktop is running.",
               opts
             );
+            console.log(
+              "Check if the post-installation steps were performed correctly as per this documentation https://docs.docker.com/engine/install/linux-postinstall/"
+            );
           }
         }
       }

package/index.js

@@ -14,7 +14,7 @@ const { findJSImports } = require("./analyzer");
 const semver = require("semver");
 const dockerLib = require("./docker");
 const binaryLib = require("./binary");
-const osQueries = require("./queries.json");
+const osQueries = require("./data/queries.json");
 const isWin = require("os").platform() === "win32";
 
 const { table } = require("table");
@@ -39,11 +39,6 @@ if (process.env.LEIN_CMD) {
   LEIN_CMD = process.env.LEIN_CMD;
 }
 
-let PIP_CMD = "pip";
-if (process.env.PIP_CMD) {
-  PIP_CMD = process.env.PIP_CMD;
-}
-
 let SWIFT_CMD = "swift";
 if (process.env.SWIFT_CMD) {
   SWIFT_CMD = process.env.SWIFT_CMD;
@@ -510,6 +505,7 @@ function addComponent(
   }
   if (!isRootPkg) {
     let pkgIdentifier = parsePackageJsonName(pkg.name);
+    let author = pkg.author || "";
     let publisher = pkg.publisher || "";
     let group = pkg.group || pkgIdentifier.scope;
     // Create empty group
@@ -573,6 +569,7 @@ function addComponent(
       return;
     }
     let component = {
+      author,
       publisher,
       group,
       name,
@@ -994,7 +991,7 @@ const createJavaBom = async (path, options) => {
     if (pomFiles && pomFiles.length) {
       const cdxMavenPlugin =
         process.env.CDX_MAVEN_PLUGIN ||
-        "org.cyclonedx:cyclonedx-maven-plugin:2.7.8";
+        "org.cyclonedx:cyclonedx-maven-plugin:2.7.9";
       const cdxMavenGoal = process.env.CDX_MAVEN_GOAL || "makeAggregateBom";
       let mvnArgs = [`${cdxMavenPlugin}:${cdxMavenGoal}`, "-DoutputName=bom"];
       if (utils.includeMavenTestScope) {
@@ -1061,9 +1058,21 @@ const createJavaBom = async (path, options) => {
             console.log(
               "Resolve the above maven error. This could be due to the following:\n"
             );
-            console.log(
-              "1. Java version requirement: cdxgen container image bundles Java 17 with maven 3.8 which might be incompatible."
-            );
+            if (
+              result.stderr &&
+              result.stderr.includes(
+                "Could not resolve dependencies" ||
+                  result.stderr.includes("no dependency information available")
+              )
+            ) {
+              console.log(
+                "1. Try building the project with 'mvn package -Dmaven.test.skip=true' using the correct version of Java and maven before invoking cdxgen."
+              );
+            } else {
+              console.log(
+                "1. Java version requirement: cdxgen container image bundles Java 19 with maven 3.9 which might be incompatible."
+              );
+            }
             console.log(
               "2. Private dependencies cannot be downloaded: Check if any additional arguments must be passed to maven and set them via MVN_ARGS environment variable."
             );
@@ -1175,7 +1184,6 @@ const createJavaBom = async (path, options) => {
         parentComponent = {
           name: rootProject,
           type: "application",
-          qualifiers: { type: "jar" },
           ...(retMap.metadata || {})
         };
         const parentPurl = decodeURIComponent(
@@ -1184,7 +1192,7 @@ const createJavaBom = async (path, options) => {
             parentComponent.group || "",
             parentComponent.name,
             parentComponent.version,
-            parentComponent.qualifiers,
+            { type: "jar" },
             null
           ).toString()
         );
@@ -1912,8 +1920,7 @@ const createNodejsBom = async (path, options) => {
  * @param options Parse options from the cli
  */
 const createPythonBom = async (path, options) => {
-  let pkgList = [];
-  let dlist = [];
+  let allImports = {};
   let metadataFilename = "";
   const pipenvMode = fs.existsSync(pathLib.join(path, "Pipfile"));
   const poetryFiles = utils.getAllFiles(
@@ -1940,7 +1947,10 @@ const createPythonBom = async (path, options) => {
     path,
     (options.multiProject ? "**/" : "") + "*.egg-info"
   );
+  let pkgList = [];
   const setupPy = pathLib.join(path, "setup.py");
+  const pyProjectFile = pathLib.join(path, "pyproject.toml");
+  const pyProjectMode = fs.existsSync(pyProjectFile);
   const requirementsMode =
     (reqFiles && reqFiles.length) || (reqDirFiles && reqDirFiles.length);
   const poetryMode = poetryFiles && poetryFiles.length;
@@ -1986,7 +1996,7 @@ const createPythonBom = async (path, options) => {
   // .egg-info files
   if (eggInfoFiles && eggInfoFiles.length) {
     for (let ef of eggInfoFiles) {
-      dlist = utils.parseBdistMetadata(
+      const dlist = utils.parseBdistMetadata(
         fs.readFileSync(ef, { encoding: "utf-8" })
       );
       if (dlist && dlist.length) {
@@ -2000,7 +2010,7 @@ const createPythonBom = async (path, options) => {
       const piplockFile = pathLib.join(path, "Pipfile.lock");
       if (fs.existsSync(piplockFile)) {
         const lockData = JSON.parse(fs.readFileSync(piplockFile));
-        dlist = await utils.parsePiplockData(lockData);
+        const dlist = await utils.parsePiplockData(lockData);
         if (dlist && dlist.length) {
           pkgList = pkgList.concat(dlist);
         }
@@ -2011,40 +2021,20 @@ const createPythonBom = async (path, options) => {
     } else if (requirementsMode) {
       metadataFilename = "requirements.txt";
       if (reqFiles && reqFiles.length) {
-        let pipWarningShown = false;
         for (let f of reqFiles) {
           const basePath = pathLib.dirname(f);
           let reqData = undefined;
-          let frozenMode = false;
-          // Attempt to pip freeze to improve precision. First try in venv mode
+          let frozen = false;
+          // Attempt to pip freeze in a virtualenv to improve precision
           if (options.installDeps) {
-            const result = spawnSync(
-              PIP_CMD,
-              ["freeze", "-r", f, "-l", "--require-virtualenv"],
-              {
-                cwd: basePath,
-                encoding: "utf-8",
-                timeout: TIMEOUT_MS
-              }
-            );
-            if (result.status === 0 && result.stdout) {
-              reqData = Buffer.from(result.stdout).toString();
-              const dlist = await utils.parseReqFile(reqData, false);
-              if (dlist && dlist.length) {
-                pkgList = pkgList.concat(dlist);
-              }
-              frozenMode = true;
-            } else if (result.status !== 0 || result.error) {
-              if (DEBUG_MODE && !pipWarningShown) {
-                pipWarningShown = true;
-                console.log(
-                  "NOTE: Setup and activate a python virtual environment for this project prior to invoking cdxgen to improve SBoM accuracy."
-                );
-              }
+            const dlist = await utils.executePipFreezeInVenv(basePath, f);
+            if (dlist && dlist.length) {
+              pkgList = pkgList.concat(dlist);
+              frozen = true;
             }
           }
           // Fallback to parsing manually
-          if (!frozenMode) {
+          if (!pkgList.length || !frozen) {
             if (DEBUG_MODE) {
               console.log(
                 `Manually parsing ${f}. The result would include only direct dependencies.`
@@ -2071,14 +2061,42 @@ const createPythonBom = async (path, options) => {
       }
     }
   }
+  // Use atom in requirements, setup.py and pyproject.toml mode
+  if (requirementsMode || setupPyMode || pyProjectMode) {
+    let dlist = undefined;
+    /**
+     * The order of preference is pyproject.toml (newer) and then setup.py
+     */
+    if (options.installDeps) {
+      if (pyProjectMode) {
+        dlist = await utils.executePipFreezeInVenv(path, pyProjectFile);
+      } else if (setupPyMode) {
+        dlist = await utils.executePipFreezeInVenv(path, setupPy);
+      }
+      if (dlist && dlist.length) {
+        pkgList = pkgList.concat(dlist);
+      }
+    }
+    // Get the imported modules and a dedupe list of packages
+    const retMap = await utils.getPyModules(path, pkgList);
+    if (retMap.pkgList && retMap.pkgList.length) {
+      pkgList = pkgList.concat(retMap.pkgList);
+    }
+    if (retMap.allImports) {
+      allImports = { ...allImports, ...retMap.allImports };
+    }
+  }
+  // Final fallback is to manually parse setup.py if we still
+  // have an empty list
   if (!pkgList.length && setupPyMode) {
     const setupPyData = fs.readFileSync(setupPy, { encoding: "utf-8" });
-    dlist = await utils.parseSetupPyFile(setupPyData);
+    const dlist = await utils.parseSetupPyFile(setupPyData);
     if (dlist && dlist.length) {
       pkgList = pkgList.concat(dlist);
     }
   }
   return buildBomNSData(options, pkgList, "pypi", {
+    allImports,
     src: path,
     filename: metadataFilename
   });
@@ -4004,6 +4022,12 @@ const createXBom = async (path, options) => {
   // python
   const pipenvMode = fs.existsSync(pathLib.join(path, "Pipfile"));
   const poetryMode = fs.existsSync(pathLib.join(path, "poetry.lock"));
+  const pyProjectMode =
+    !poetryMode && fs.existsSync(pathLib.join(path, "pyproject.toml"));
+  const setupPyMode = fs.existsSync(pathLib.join(path, "setup.py"));
+  if (pipenvMode || poetryMode || pyProjectMode || setupPyMode) {
+    return await createPythonBom(path, options);
+  }
   const reqFiles = utils.getAllFiles(
     path,
     (options.multiProject ? "**/" : "") + "*requirements*.txt"
@@ -4012,21 +4036,13 @@ const createXBom = async (path, options) => {
     path,
     (options.multiProject ? "**/" : "") + "requirements/*.txt"
   );
-  const setupPy = pathLib.join(path, "setup.py");
   const requirementsMode =
     (reqFiles && reqFiles.length) || (reqDirFiles && reqDirFiles.length);
   const whlFiles = utils.getAllFiles(
     path,
     (options.multiProject ? "**/" : "") + "*.whl"
   );
-  const setupPyMode = fs.existsSync(setupPy);
-  if (
-    requirementsMode ||
-    pipenvMode ||
-    poetryMode ||
-    setupPyMode ||
-    whlFiles.length
-  ) {
+  if (requirementsMode || whlFiles.length) {
     return await createPythonBom(path, options);
   }
   // go

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cyclonedx/cdxgen",
-  "version": "8.5.2",
+  "version": "8.6.0",
   "description": "Creates CycloneDX Software Bill-of-Materials (SBOM) from source or container image",
   "homepage": "http://github.com/cyclonedx/cdxgen",
   "author": "Prabhu Subramanian <prabhu@appthreat.com>",
@@ -36,7 +36,7 @@
     "test": "jest --inject-globals false",
     "watch": "jest --watch --inject-globals false",
     "lint": "eslint index.js utils.js binary.js server.js docker.js *.test.js bin/cdxgen",
-    "pretty": "prettier --write *.js bin/cdxgen --trailing-comma=none"
+    "pretty": "prettier --write *.js data/*.json bin/cdxgen --trailing-comma=none"
   },
   "engines": {
     "node": ">=12.0.0"
@@ -49,8 +49,8 @@
     "url": "https://github.com/cyclonedx/cdxgen/issues"
   },
   "dependencies": {
-    "@babel/parser": "^7.21.8",
-    "@babel/traverse": "^7.21.5",
+    "@babel/parser": "^7.22.5",
+    "@babel/traverse": "^7.22.5",
     "cheerio": "^1.0.0-rc.12",
     "edn-data": "^1.0.0",
     "glob": "^8.1.0",
@@ -63,16 +63,17 @@
     "parse-packagejson-name": "^1.0.1",
     "prettify-xml": "^1.2.0",
     "properties-reader": "^2.2.0",
-    "semver": "^7.5.0",
+    "semver": "^7.5.1",
     "ssri": "^8.0.1",
     "table": "^6.8.1",
-    "tar": "^6.1.14",
+    "tar": "^6.1.15",
     "uuid": "^9.0.0",
     "xml-js": "^1.6.11",
     "xmlbuilder": "^15.1.1",
     "yargs": "^17.7.2"
   },
   "optionalDependencies": {
+    "@appthreat/atom": "^0.10.1",
     "@cyclonedx/cdxgen-plugins-bin": "^1.1.0",
     "body-parser": "^1.20.2",
     "compression": "^1.7.4",
@@ -81,14 +82,10 @@
   "files": [
     "*.js",
     "bin/",
-    "spdx-licenses.json",
-    "lic-mapping.json",
-    "known-licenses.json",
-    "vendor-alias.json",
-    "queries.json"
+    "data/"
   ],
   "devDependencies": {
-    "eslint": "^8.40.0",
+    "eslint": "^8.42.0",
     "jest": "^29.5.0"
   }
 }