@cyclonedx/cdxgen 10.11.0 → 11.0.0

package/README.md

@@ -33,6 +33,7 @@ Our philosophy:
 - Precision: Try using multiple techniques to improve precision, even if it takes extra time.
 - Personas: Cater to the needs of a range of personas such as security researchers, compliance auditors, developers, and SOC.
 - Lifecycle: Support BOM generation for various product lifecycles.
+- Machine Learning: Optimize the generated data for Machine Learning (ML) purposes by considering the various model properties.
 
 ## Documentation
 
@@ -92,7 +93,7 @@ docker run --rm -e CDXGEN_DEBUG_MODE=debug -v /tmp:/tmp -v $(pwd):/app:rw -t ghc
 In deno applications, cdxgen could be directly imported without any conversion. Please see the section on [integration as a library](#integration-as-library)
 
 ```ts
-import { createBom, submitBom } from "npm:@cyclonedx/cdxgen@^10.9.6";
+import { createBom, submitBom } from "npm:@cyclonedx/cdxgen@^11.0.0";
 ```
 
 ## Getting Help
@@ -116,6 +117,7 @@ Options:
       --deep                   Perform deep searches for components. Useful while scanning C/C++ apps, live OS and oci i
                                mages.                                                                          [boolean]
       --server-url             Dependency track url. Eg: https://deptrack.cyclonedx.io
+      --skip-dt-tls-check      Skip TLS certificate check when calling Dependency-Track.      [boolean] [default: false]
       --api-key                Dependency track api key
       --project-group          Dependency track project group
       --project-name           Dependency track project name. Default use the directory name
@@ -137,7 +139,7 @@ Options:
       --validate               Validate the generated SBOM using json schema. Defaults to true. Pass --no-validate to di
                                sable.                                                          [boolean] [default: true]
       --evidence               Generate SBOM with evidence for supported languages.           [boolean] [default: false]
-      --spec-version           CycloneDX Specification version to use. Defaults to 1.5           [number] [default: 1.5]
+      --spec-version           CycloneDX Specification version to use. Defaults to 1.6           [number] [default: 1.6]
       --filter                 Filter components containing this word in purl or component.properties.value. Multiple va
                                lues allowed.                                                                     [array]
       --only                   Include components only containing this word in purl. Useful to generate BOM with first p
@@ -145,17 +147,22 @@ Options:
       --author                 The person(s) who created the BOM. Set this value if you're intending the modify the BOM
                                and claim authorship.                               [array] [default: "OWASP Foundation"]
       --profile                BOM profile to use for generation. Default generic.
-  [choices: "appsec", "research", "operational", "threat-modeling", "license-compliance", "generic"] [default: "generic"
-                                                                                                                       ]
+  [choices: "appsec", "research", "operational", "threat-modeling", "license-compliance", "generic", "machine-learning",
+                                                       "ml", "deep-learning", "ml-deep", "ml-tiny"] [default: "generic"]
       --exclude                Additional glob pattern(s) to ignore                                              [array]
-      --include-formulation    Generate formulation section with git metadata and build tools. Defaults to true. Invoke
-                               with --no-include-formulation to disable.                       [boolean] [default: true]
+      --include-formulation    Generate formulation section with git metadata and build tools. Defaults to false.
+                                                                                              [boolean] [default: false]
       --include-crypto         Include crypto libraries as components.                        [boolean] [default: false]
       --standard               The list of standards which may consist of regulations, industry or organizational-specif
                                ic standards, maturity models, best practices, or any other requirements which can be eva
                                luated against or attested to.
   [array] [choices: "asvs-4.0.3", "bsimm-v13", "masvs-2.0.0", "nist_ssdf-1.1", "pcissc-secure-slc-1.1", "scvs-1.0.0", "s
                                                                                                      saf-DRAFT-2023-11"]
+      --min-confidence         Minimum confidence needed for the identity of a component from 0 - 1, where 1 is 100% con
+                               fidence.                                                            [number] [default: 0]
+      --technique              Analysis technique to use
+  [array] [choices: "auto", "source-code-analysis", "binary-analysis", "manifest-analysis", "hash-comparison", "instrume
+                                                                                                   ntation", "filename"]
       --auto-compositions      Automatically set compositions when the BOM was filtered. Defaults to true
                                                                                                [boolean] [default: true]
   -h, --help                   Show help                                                                       [boolean]

package/bin/cdxgen.js

@@ -235,6 +235,11 @@ const args = yargs(hideBin(process.argv))
       "threat-modeling",
       "license-compliance",
       "generic",
+      "machine-learning",
+      "ml",
+      "deep-learning",
+      "ml-deep",
+      "ml-tiny",
     ],
   })
   .option("lifecycle", {
@@ -292,6 +297,24 @@ const args = yargs(hideBin(process.argv))
     hidden: true,
     choices: ["safe-pip-install", "suggest-build-tools"],
   })
+  .option("min-confidence", {
+    description:
+      "Minimum confidence needed for the identity of a component from 0 - 1, where 1 is 100% confidence.",
+    default: 0,
+    type: "number",
+  })
+  .option("technique", {
+    description: "Analysis technique to use",
+    choices: [
+      "auto",
+      "source-code-analysis",
+      "binary-analysis",
+      "manifest-analysis",
+      "hash-comparison",
+      "instrumentation",
+      "filename",
+    ],
+  })
   .completion("completion", "Generate bash/zsh completion")
   .array("type")
   .array("excludeType")
@@ -301,6 +324,7 @@ const args = yargs(hideBin(process.argv))
   .array("exclude")
   .array("standard")
   .array("feature-flags")
+  .array("technique")
   .option("auto-compositions", {
     type: "boolean",
     default: true,
@@ -313,6 +337,14 @@ const args = yargs(hideBin(process.argv))
       "$0 -t java -t js .",
       "Generate a SBOM for Java and JavaScript in the current directory",
     ],
+    [
+      "$0 -t java --profile ml .",
+      "Generate a Java SBOM for machine learning purposes.",
+    ],
+    [
+      "$0 -t python --profile research .",
+      "Generate a Python SBOM for appsec research.",
+    ],
     ["$0 --server", "Run cdxgen as a server"],
   ])
   .epilogue("for documentation, visit https://cyclonedx.github.io/cdxgen")
@@ -414,6 +446,29 @@ const applyAdvancedOptions = (options) => {
     case "license-compliance":
       process.env.FETCH_LICENSE = "true";
       break;
+    case "ml-tiny":
+      process.env.FETCH_LICENSE = "true";
+      options.deep = false;
+      options.evidence = false;
+      options.includeCrypto = false;
+      options.installDeps = false;
+      break;
+    case "machine-learning":
+    case "ml":
+      process.env.FETCH_LICENSE = "true";
+      options.deep = true;
+      options.evidence = false;
+      options.includeCrypto = false;
+      options.installDeps = true;
+      break;
+    case "deep-learning":
+    case "ml-deep":
+      process.env.FETCH_LICENSE = "true";
+      options.deep = true;
+      options.evidence = true;
+      options.includeCrypto = true;
+      options.installDeps = true;
+      break;
     default:
       break;
   }
@@ -685,8 +740,10 @@ const checkPermissions = (filePath) => {
       usagesSlicesFile: options.usagesSlicesFile,
       dataFlowSlicesFile: options.dataFlowSlicesFile,
       reachablesSlicesFile: options.reachablesSlicesFile,
+      semanticsSlicesFile: options.semanticsSlicesFile,
       includeCrypto: options.includeCrypto,
       specVersion: options.specVersion,
+      profile: options.profile,
     };
     const dbObjMap = await evinserModule.prepareDB(evinseOptions);
     if (dbObjMap) {
@@ -736,7 +793,6 @@ const checkPermissions = (filePath) => {
     printTable(bomNSData.bomJson);
     // CBOM related print
     if (options.includeCrypto) {
-      console.log("\n*** Cryptography BOM ***");
       printTable(bomNSData.bomJson, ["cryptographic-asset"]);
       printDependencyTree(bomNSData.bomJson, "provides");
     }

package/bin/repl.js

@@ -161,7 +161,7 @@ cdxgenRepl.defineCommand("search", {
           let dependenciesSearchStr = searchStr;
           if (!searchStr.includes("~>")) {
             dependenciesSearchStr = `dependencies[ref ~> /${searchStr}/i or dependsOn ~> /${searchStr}/i or provides ~> /${searchStr}/i]`;
-            searchStr = `components[group ~> /${searchStr}/i or name ~> /${searchStr}/i or description ~> /${searchStr}/i or publisher ~> /${searchStr}/i or purl ~> /${searchStr}/i]`;
+            searchStr = `components[group ~> /${searchStr}/i or name ~> /${searchStr}/i or description ~> /${searchStr}/i or publisher ~> /${searchStr}/i or purl ~> /${searchStr}/i or tags ~> /${searchStr}/i]`;
           }
           const expression = jsonata(searchStr);
           let components = await expression.evaluate(sbom);

package/data/README.md

@@ -22,4 +22,5 @@ Contents of data directory and their purpose.
 | wrapdb-releases.json  | Database of all available meson wraps. Generated using contrib/wrapdb.py.                                |
 | frameworks-list.json  | List of string fragments to categorize components into frameworks                                        |
 | crypto-oid.json       | Peter Gutmann's crypto oid [mapping](https://www.cs.auckland.ac.nz/~pgut001). GPL, BSD, or CC BY license |
-| glibc-stdlib.json     | Standard libraries that can be filtered out in C++ |
+| glibc-stdlib.json     | Standard libraries that can be filtered out in C++                                                       |
+| component-tags.json   | List of tags to extract from component description text for easy classification.                         |

package/data/component-tags.json

@@ -0,0 +1,325 @@
+{
+  "description": {
+    "all": [
+      "sql",
+      "xml",
+      "web",
+      "security",
+      "database",
+      "json",
+      "yaml",
+      "validation",
+      "sanitization",
+      "cloud",
+      "iam",
+      "auth",
+      "middleware",
+      "serialization",
+      "event",
+      "stream",
+      "rpc",
+      "socket",
+      "proto",
+      "resource",
+      "sensitive",
+      "template",
+      "log",
+      "logging",
+      "service",
+      "api",
+      "slf4j",
+      "parse",
+      "emit",
+      "jdbc",
+      "connect",
+      "pool",
+      "beans",
+      "transaction",
+      "mysql",
+      "postgres",
+      "oracle",
+      "mongo",
+      "redis",
+      "splunk",
+      "stripe",
+      "payment",
+      "finance",
+      "currency",
+      "coin",
+      "monero",
+      "ssl",
+      "traffic",
+      "mvc",
+      "html",
+      "escape",
+      "unescape",
+      "rest",
+      "tomcat",
+      "hibernate",
+      "orm",
+      "aop",
+      "jwt",
+      "saml",
+      "token",
+      "tls",
+      "codec",
+      "cron",
+      "crypto",
+      "jce",
+      "certificate",
+      "developer",
+      "tools",
+      "autoconfigure",
+      "test",
+      "jsonpath",
+      "bytecode",
+      "mock",
+      "injection",
+      "comparators",
+      "transform",
+      "encode",
+      "decode",
+      "ldap",
+      "owasp",
+      "fileupload",
+      "beanshell",
+      "spel",
+      "mail",
+      "apacheds",
+      "jndi",
+      "ldif",
+      "jdbm",
+      "kerberos",
+      "oidc",
+      "oauth2",
+      "cli",
+      "binary",
+      "ml",
+      "ai",
+      "azure",
+      "gcp",
+      "terraform",
+      "redis",
+      "valkey",
+      "lint",
+      "bundle",
+      "object-persistence",
+      "text-to-image",
+      "translat",
+      "object-detect",
+      "mvc",
+      "framework",
+      "graph",
+      "templates",
+      "fastjson",
+      "simd",
+      "event-driven",
+      "productivity",
+      "typesafe",
+      "projections",
+      "performance",
+      "plugins",
+      "non-block",
+      "microsoft"
+    ]
+  },
+  "properties": {
+    "all": [
+      "sql",
+      "http",
+      "xml",
+      "cloud",
+      "middleware",
+      "framework",
+      "bluetooth",
+      "wifi",
+      "wireless",
+      "driver",
+      "graphics",
+      "firmware",
+      "gyroscope",
+      "accelerometer",
+      "mobile",
+      "network",
+      "battery",
+      "matrix",
+      "thunderbolt",
+      "crypto",
+      "algorithm",
+      "encrypt",
+      "decrypt",
+      "registry",
+      "payment",
+      "stripe",
+      "apple-pay",
+      "icloud"
+    ],
+    "obom": [
+      "windows_drivers",
+      "windows_patches",
+      "windows_programs",
+      "processor",
+      "services_snapshot",
+      "apt_sources",
+      "behavioral_reverse_shell",
+      "certificates",
+      "chrome_extensions",
+      "crontab_snapshot",
+      "deb_packages",
+      "docker_container_ports",
+      "docker_containers",
+      "docker_networks",
+      "docker_volumes",
+      "etc_hosts",
+      "firefox_addons",
+      "vscode_extensions",
+      "homebrew_packages",
+      "installed_applications",
+      "interface_addresses",
+      "kernel_info",
+      "kernel_integrity",
+      "kernel_modules",
+      "ld_preload",
+      "listening_ports",
+      "os_version",
+      "pipes",
+      "pipes_snapshot",
+      "portage_packages",
+      "process_events",
+      "processes",
+      "python_packages",
+      "rpm_packages",
+      "scheduled_tasks",
+      "services_snapshot",
+      "startup_items",
+      "system_info_snapshot",
+      "windows_drivers",
+      "windows_patches",
+      "windows_programs",
+      "windows_shared_resources",
+      "yum_sources",
+      "appcompat_shims",
+      "browser_plugins",
+      "certificates",
+      "chocolatey_packages",
+      "chrome_extensions",
+      "etc_hosts",
+      "firefox_addons",
+      "ie_extensions",
+      "kernel_info",
+      "npm_packages",
+      "opera_extensions",
+      "pipes_snapshot",
+      "process_open_sockets",
+      "safari_extensions",
+      "scheduled_tasks",
+      "services_snapshot",
+      "startup_items",
+      "routes",
+      "system_info_snapshot",
+      "win_version",
+      "windows_firewall_rules",
+      "windows_optional_features",
+      "windows_programs",
+      "windows_shared_resources",
+      "windows_update_history",
+      "wmi_cli_event_consumers",
+      "wmi_cli_event_consumers_snapshot",
+      "wmi_event_filters",
+      "wmi_filter_consumer_binding"
+    ]
+  },
+  "name": {
+    "sbom": [
+      { "test": ["(junit|xmlunit|testng|chai|mocha|jest|test4j)"] },
+      {
+        "security": ["(boringssl|openssl|libressl|libssl|gnutls|jose|keyutils)"]
+      },
+      { "native": ["(ffi|native)"] },
+      { "parse": ["(parser)"] },
+      { "transform": ["(transformer)"] }
+    ],
+    "obom": [
+      {
+        "devel": [
+          "-(dev|devel|headers|sdk|libs|extension|headers+x86|headers+x64|headers+arm64)$",
+          "^(git)[-]?",
+          "^(sdk|windows+sdk)"
+        ]
+      },
+      {
+        "bin": [
+          "(-bin|redistributable|clickonce|bootstrappermsi|bootstrappermsires|clickoncesigntoolmsi|codecoveragemsi|msires|sharedmsi|x64msi|arm64msi|sharedmsi|x64vmsi|filehandler_amd64|filehandler_x86|protocolhandlermsi|interopmsi|interopx64msi|shellmsires|shellx64msi)$"
+        ]
+      },
+      { "kernel": ["^(linux|kernel|os-image)"] },
+      {
+        "security": [
+          "(selinux|apparmor|security|boringssl|openssl|libressl|gnutls|jose|keyutils|passwd|libssl|libaudit|gcrypt|libpam|libseccomp)"
+        ]
+      },
+      {
+        "container": [
+          "(container|podman|docker|runc|nerdctl|crun|libvirt|qemu)"
+        ]
+      },
+      {
+        "build": [
+          "(cpp|fortran|gcc|make|meson|bazel|maven|gradle|sbt|ant|gdb|boost|compiler|kotlin|cargo|rustc|llvm|libstdc)"
+        ]
+      },
+      {
+        "network": [
+          "(tailscale|wireguard|openvpn|dns|cockpit|cups|dhcp|network|iproute|iptables|mosh|netavark|openssh|rsync|tcpdump|libssh)"
+        ]
+      },
+      { "webserver": ["(httpd|http2|tomcat|jboss)"] },
+      {
+        "crypto": [
+          "(crypt|gpg|keys|certificates|gnupg|certifi|pubkey|keyutils|nss|keyring)"
+        ]
+      },
+      { "repository": ["(-repos|-release|ostree|appstream)"] },
+      { "shell": ["(bash|zsh|csh|fish|binsh|dash|oilsh)"] },
+      { "bluetooth": ["(bluez|bluetooth)"] },
+      { "sound": ["(alsa|pulseaudio|wireplumber|flac|codecs|ldac|sound)"] },
+      {
+        "compression": [
+          "(brotli|xz-utils|zstd|lz4|zlib|bz2|lzma5|bzip2|libarchive)",
+          "(tar|zip|webp|zchunk)$"
+        ]
+      },
+      {
+        "runtime": [
+          "(perl|lua|php|python|ruby|dotnet|java|swift|runtime|glibc|libc6|musl|wasm|\\.net|asp\\.net|node.js|node|groovy)"
+        ]
+      },
+      { "editor": ["(vim|emacs|nano|hexedit)"] },
+      { "xml": ["(xml|expat)"] },
+      { "boot": ["(grub|systemd-boot|syslinux|init-system|sysvinit)"] },
+      {
+        "gui": [
+          "(wayland|xorg|X11|mesa|vulkan|tk|wkhtmltox|electron|Xrender|glib2)"
+        ]
+      },
+      {
+        "package": [
+          "(rpm|dnf|yum|apt|zypper|apk|conda|dpkg|dnf5)$",
+          "^(conda_package_|conda-package-|libapt|dnf5-|libdnf5)"
+        ]
+      },
+      {
+        "browser": [
+          "^(edge)",
+          "(firefox|chrome|opera|brave|mullvad|tor|chromium)",
+          "(microsoft+edge|microsoft+edge+webview2|microsoft+html)"
+        ]
+      },
+      {
+        "chat": ["(webex|teams|slack|discord|vesktop|matrix|signal|whatsapp)"]
+      },
+      { "logging": ["(log4j|logging|slf4j)"] },
+      { "root": ["^(sudo|systemd|pam|shadow)$"] }
+    ]
+  }
+}

package/lib/evinser/evinser.js

@@ -17,6 +17,7 @@ import {
   getMavenCommand,
   getTimestamp,
 } from "../helpers/utils.js";
+import { postProcess } from "../stages/postgen/postgen.js";
 import { createSemanticsSlices } from "./swiftsem.js";
 
 const DB_NAME = "evinser.db";
@@ -1179,6 +1180,8 @@ export const createEvinseFile = (sliceArtefacts, options) => {
   const evinseOutFile = options.output;
   const bomJson = JSON.parse(fs.readFileSync(bomFile, "utf8"));
   const components = bomJson.components || [];
+  // Clear existing annotations
+  bomJson.annotations = [];
   let occEvidencePresent = false;
   let csEvidencePresent = false;
   let servicesPresent = false;
@@ -1261,9 +1264,6 @@ export const createEvinseFile = (sliceArtefacts, options) => {
     bomJson.dependencies = newDependencies;
   }
   if (options.annotate) {
-    if (!bomJson.annotations) {
-      bomJson.annotations = [];
-    }
     if (usagesSlicesFile && fs.existsSync(usagesSlicesFile)) {
       bomJson.annotations.push({
         subjects: [bomJson.serialNumber],
@@ -1294,7 +1294,11 @@ export const createEvinseFile = (sliceArtefacts, options) => {
   // Set the current timestamp to indicate this is newer
   bomJson.metadata.timestamp = getTimestamp();
   delete bomJson.signature;
-  fs.writeFileSync(evinseOutFile, JSON.stringify(bomJson, null, null));
+  const bomNSData = postProcess({ bomJson }, options);
+  fs.writeFileSync(
+    evinseOutFile,
+    JSON.stringify(bomNSData.bomJson, null, null),
+  );
   if (occEvidencePresent || csEvidencePresent || servicesPresent) {
     console.log(evinseOutFile, "created successfully.");
   } else {
@@ -1305,7 +1309,8 @@ export const createEvinseFile = (sliceArtefacts, options) => {
   if (tempDir?.startsWith(tmpdir())) {
     fs.rmSync(tempDir, { recursive: true, force: true });
   }
-  return bomJson;
+  // Redo post processing with evinse data
+  return bomNSData?.bomJson;
 };
 
 /**

package/lib/helpers/display.js

@@ -36,12 +36,13 @@ export function printTable(
     columnDefault: {
       width: 30,
     },
-    columnCount: 4,
+    columnCount: 5,
     columns: [
       { width: 25 },
       { width: 35 },
       { width: 25, alignment: "right" },
       { width: 15 },
+      { width: 25 },
     ],
   };
   const stream = createStream(config);
@@ -52,6 +53,7 @@ export function printTable(
     "Name",
     filterTypes?.includes("cryptographic-asset") ? "Version / oid" : "Version",
     "Scope",
+    "Tags",
   ]);
   for (const comp of bomJson.components) {
     if (filterTypes && !filterTypes.includes(comp.type)) {
@@ -63,6 +65,7 @@ export function printTable(
         comp.name,
         `\x1b[1;35m${comp.cryptoProperties?.oid || ""}\x1b[0m`,
         comp.scope || "",
+        (comp.tags || []).join(", "),
       ]);
     } else {
       stream.write([
@@ -70,6 +73,7 @@ export function printTable(
         highlightStr(comp.name, highlight),
         `\x1b[1;35m${comp.version || ""}\x1b[0m`,
         comp.scope || "",
+        (comp.tags || []).join(", "),
       ]);
     }
   }
@@ -98,16 +102,17 @@ export function printOSTable(bomJson) {
     columnDefault: {
       width: 50,
     },
-    columnCount: 3,
-    columns: [{ width: 20 }, { width: 40 }, { width: 50 }],
+    columnCount: 4,
+    columns: [{ width: 20 }, { width: 40 }, { width: 50 }, { width: 25 }],
   };
   const stream = createStream(config);
-  stream.write(["Type", "Title", "Properties"]);
+  stream.write(["Type", "Title", "Properties", "Tags"]);
   for (const comp of bomJson.components) {
     stream.write([
       comp.type,
       `\x1b[1;35m${comp.name.replace(/\+/g, " ").replace(/--/g, "::")}\x1b[0m`,
       formatProps(comp.properties || []),
+      (comp.tags || []).join(", "),
     ]);
   }
   console.log();
@@ -436,17 +441,23 @@ export function printSummary(bomJson) {
       alignment: "center",
       content: "BOM summary",
     },
+    columns: [{ wrapWord: true, width: 100 }],
   };
   const metadataProperties = bomJson?.metadata?.properties;
   if (!metadataProperties) {
     return;
   }
-  const tools = bomJson?.metadata?.tools?.components;
   let message = "";
   let bomPkgTypes = [];
   let bomPkgNamespaces = [];
+  // Print any annotations found
+  const annotations = bomJson?.annotations || [];
+  for (const annot of annotations) {
+    message = `${message}\n${annot.text}`;
+  }
+  const tools = bomJson?.metadata?.tools?.components;
   if (tools) {
-    message = "** Generator Tools **";
+    message = `${message}\n\n** Generator Tools **`;
     for (const atool of tools) {
       if (atool.name && atool.version) {
         message = `${message}\n${atool.name} (${atool.version})`;