@cdxgen/cdxgen 13.2.0 → 13.4.0

package/lib/cli/nativeBom.js

@@ -1,7 +1,7 @@
 import { Buffer } from "node:buffer";
 import { readFileSync, statSync } from "node:fs";
 import { homedir } from "node:os";
-import { basename, dirname, join, relative, resolve } from "node:path";
+import { basename, dirname, join, relative, resolve, sep } from "node:path";
 import process from "node:process";
 
 import { build } from "@cdxgen/cdx-purl";
@@ -28,12 +28,19 @@ import {
 } from "../core/fs.js";
 import { thoughtLog } from "../core/logger.js";
 import { dirNameStr, isMac } from "../core/paths.js";
+import { readCrateArchiveValues } from "../ecosystems/cargoCache.js";
 import {
   buildDependentPurl,
   collapseCmakeVersions,
+  preferFetchedCmakeDependencies,
   resolveCmakeContext,
+  withoutScrapedFetchDependency,
 } from "../ecosystems/cmakeResolver.js";
-import { getCppModules } from "../ecosystems/cppEvidence.js";
+import {
+  emptyCppBuildContext,
+  resolveCppBuildContext,
+} from "../ecosystems/cppBuildContext.js";
+import { getCppModules, hasCppSources } from "../ecosystems/cppEvidence.js";
 import {
   getCratesMetadata,
   getSwiftPackageMetadata,
@@ -48,6 +55,7 @@ import {
 } from "../ecosystems/parsers-elm.js";
 import { parseGleamProject } from "../ecosystems/parsers-gleam.js";
 import {
+  isGoModWhyNotNeeded,
   parseGoListDep,
   parseGoModData,
   parseGoModGraph,
@@ -55,6 +63,7 @@ import {
   parseGoModWhy,
   parseGopkgData,
   parseGosumData,
+  parseGosumHashes,
 } from "../ecosystems/parsers-go.js";
 import { parseJuliaProject } from "../ecosystems/parsers-julia.js";
 import {
@@ -69,6 +78,7 @@ import {
 } from "../ecosystems/parsers-lua.js";
 import {
   buildObjectForCocoaPod,
+  classifySwiftDependencyScopes,
   executeAlpmList,
   executeApkList,
   executeDpkgList,
@@ -111,7 +121,9 @@ import {
 } from "../ecosystems/parsers-rust.js";
 import { parseSpackLock } from "../ecosystems/parsers-spack.js";
 import { parseStackProject } from "../ecosystems/parsers-stack.js";
+import { findVendoredCode } from "../ecosystems/vendoredCode.js";
 import { resolveZigGraph } from "../ecosystems/zigResolver.js";
+import { cmakeBuildDirCandidates } from "../inventory/cmakeBuildDirs.js";
 import { mergeDependencies, trimComponents } from "../inventory/depsUtils.js";
 import { convertOSQueryResults } from "../inventory/evidenceUtils.js";
 import { applyPurl, nixBomRef, tryBuildPurl } from "../inventory/purl.js";
@@ -196,7 +208,6 @@ export function getCargoCacheDir() {
 export async function createGoBom(path, options) {
   let pkgList = [];
   let dependencies = [];
-  const allImports = {};
   let parentComponent = createDefaultParentComponent(path, "golang", options);
   // Is this a binary file
   let maybeBinary;
@@ -236,6 +247,8 @@ export async function createGoBom(path, options) {
       }
     }
     const doneList = {};
+    const requiredModules = new Set();
+    const unneededModules = new Set();
     let circuitBreak = false;
     if (DEBUG_MODE) {
       console.log(
@@ -249,12 +262,12 @@ export async function createGoBom(path, options) {
       }
       const pkgFullName = `${apkg.name}`;
       if (apkg.scope === "required") {
-        allImports[pkgFullName] = true;
+        requiredModules.add(pkgFullName);
         continue;
       }
       if (
         apkg.scope === "optional" ||
-        allImports[pkgFullName] ||
+        requiredModules.has(pkgFullName) ||
         doneList[pkgFullName]
       ) {
         continue;
@@ -283,19 +296,40 @@ export async function createGoBom(path, options) {
         const mstdout = mresult.stdout;
         if (mstdout) {
           const cmdOutput = Buffer.from(mstdout).toString();
-          const whyPkg = parseGoModWhy(cmdOutput);
-          // whyPkg would include this package string
-          // github.com/golang/protobuf/proto github.com/golang/protobuf
-          // golang.org/x/tools/cmd/goimports golang.org/x/tools
-          if (whyPkg?.includes(pkgFullName)) {
-            allImports[pkgFullName] = true;
+          if (isGoModWhyNotNeeded(cmdOutput)) {
+            unneededModules.add(pkgFullName);
+          } else {
+            // The last line of the import chain is a package of the module:
+            // github.com/golang/protobuf/proto for github.com/golang/protobuf
+            const whyPkg = parseGoModWhy(cmdOutput);
+            if (
+              whyPkg === pkgFullName ||
+              whyPkg?.startsWith(`${pkgFullName}/`)
+            ) {
+              requiredModules.add(pkgFullName);
+            }
           }
           doneList[pkgFullName] = true;
         }
       }
     }
+    // A module go reports as unneeded is optional, but only when every module
+    // was checked: once go mod why fails, the answers are incomplete, and a
+    // tool failure is not evidence that a module is unused.
+    for (const apkg of pkgList) {
+      if (apkg.scope) {
+        continue;
+      }
+      if (requiredModules.has(apkg.name)) {
+        apkg.scope = "required";
+      } else if (!circuitBreak && unneededModules.has(apkg.name)) {
+        apkg.scope = "optional";
+      }
+    }
     if (DEBUG_MODE) {
-      console.log(`Required packages: ${Object.keys(allImports).length}`);
+      console.log(
+        `Required packages: ${requiredModules.size}, not needed: ${circuitBreak ? "unknown" : unneededModules.size}`,
+      );
     }
     return buildBomNSData(options, pkgList, "golang", {
       src: path,
@@ -313,12 +347,9 @@ export async function createGoBom(path, options) {
         console.log(`Parsing ${f}`);
       }
       const gosumData = readFileSync(f, { encoding: "utf-8" });
-      const dlist = await parseGosumData(gosumData);
-      if (dlist?.length) {
-        dlist.forEach((pkg) => {
-          gosumMap[`${pkg.name}@${pkg.version}`] = pkg._integrity;
-        });
-      }
+      // Only the hashes are needed here; reading them costs no registry
+      // round trip. The components come from go.mod and the go list output.
+      Object.assign(gosumMap, parseGosumHashes(gosumData));
     }
   }
 
@@ -560,7 +591,6 @@ export async function createGoBom(path, options) {
       }
       if (pkgList.length && !shouldManuallyParse) {
         return buildBomNSData(options, pkgList, "golang", {
-          allImports,
           dependencies,
           parentComponent,
           src: path,
@@ -1138,7 +1168,9 @@ export function buildCargoCacheComponent(crateFile) {
 
 /**
  * Enriches a cargo cache component with a SHA-256 hash and filename-based
- * identity evidence (confidence 0.5). Hash computation failures are ignored.
+ * identity evidence (confidence 0.5), and with what the archive's own manifest
+ * says: its licence, description and repository, read from the archive without
+ * extracting it anywhere. Hash computation failures are ignored.
  *
  * @param {string} crateFile Absolute path to the `.crate` archive
  * @param {object} [component] Component produced by `buildCargoCacheComponent`
@@ -1158,6 +1190,33 @@ export async function enrichCargoCacheComponent(crateFile, component) {
   } catch {
     // continue without hashes
   }
+  const archive = await readCrateArchiveValues(
+    crateFile,
+    component.name,
+    component.version,
+  );
+  if (archive) {
+    if (archive.values.license && !component.license) {
+      component.license = archive.values.license;
+    }
+    if (archive.values.licenses && !component.licenses) {
+      component.licenses = archive.values.licenses;
+    }
+    if (archive.values.description && !component.description) {
+      component.description = archive.values.description;
+    }
+    if (archive.values.repository && !component.repository) {
+      component.repository = { url: archive.values.repository };
+    }
+    if (archive.values.homepage && !component.homepage) {
+      component.homepage = { url: archive.values.homepage };
+    }
+    component.properties = component.properties || [];
+    component.properties.push({
+      name: "cdx:cargo:metadataSource",
+      value: "local-registry",
+    });
+  }
   component.evidence = {
     identity: {
       field: "purl",
@@ -1340,6 +1399,7 @@ export function createCppBom(path, options) {
   }
   let pkgList = [];
   let parentComponentDependencies = [];
+  const subProjects = [];
   if (conanLockFiles.length) {
     for (const f of conanLockFiles) {
       if (DEBUG_MODE) {
@@ -1415,22 +1475,31 @@ export function createCppBom(path, options) {
   }
   if (cmakeLikeFiles.length) {
     const beforeCmake = pkgList.length;
+    // The files at the scan root name the project, so they are read before
+    // the files of any sub-project (the sort is stable otherwise). The scan
+    // root may be given as a relative path while the files come back absolute.
+    const scanRoot = resolve(path);
+    const atScanRoot = (f) => resolve(dirname(f)) === scanRoot;
+    cmakeLikeFiles.sort(
+      (a, b) => (atScanRoot(a) ? 0 : 1) - (atScanRoot(b) ? 0 : 1),
+    );
     for (const f of cmakeLikeFiles) {
       if (DEBUG_MODE) {
         console.log(`Parsing ${f}`);
       }
-      const basePath = dirname(f);
       const retMap = parseCmakeLikeFile(f, "generic");
       if (retMap.pkgList?.length) {
         pkgList = pkgList.concat(retMap.pkgList);
       }
       if (
-        basePath === path &&
+        atScanRoot(f) &&
         retMap.parentComponent &&
         Object.keys(retMap.parentComponent).length
       ) {
         if (!parentComponent) {
           parentComponent = retMap.parentComponent;
+          // a sub-project reusing the root project's name is the root itself
+          addedParentComponentsMap[retMap.parentComponent.name] = true;
         } else {
           parentComponent.components = parentComponent.components || [];
           if (!addedParentComponentsMap[retMap.parentComponent.name]) {
@@ -1443,8 +1512,18 @@ export function createCppBom(path, options) {
         Object.keys(retMap.parentComponent).length &&
         !addedParentComponentsMap[retMap.parentComponent.name]
       ) {
+        // a project() below the scan root is a part of this project, not a
+        // dependency of it
         retMap.parentComponent.type = "library";
-        pkgList.push(retMap.parentComponent);
+        const subprojectDir = relative(scanRoot, resolve(dirname(f)))
+          .split(sep)
+          .join("/");
+        retMap.parentComponent.properties = [
+          ...(retMap.parentComponent.properties || []),
+          { name: "cdx:cmake:subprojectDir", value: subprojectDir || "." },
+        ];
+        subProjects.push(retMap.parentComponent);
+        addedParentComponentsMap[retMap.parentComponent.name] = true;
       }
       // Retain the dependency tree from cmake
       if (retMap.dependenciesList) {
@@ -1464,7 +1543,11 @@ export function createCppBom(path, options) {
     // resolved versions and are left as they are.
     pkgList = pkgList
       .slice(0, beforeCmake)
-      .concat(collapseCmakeVersions(pkgList.slice(beforeCmake)));
+      .concat(
+        collapseCmakeVersions(
+          preferFetchedCmakeDependencies(pkgList.slice(beforeCmake)),
+        ),
+      );
   }
   // CMake describes how a project is built from source. An image or an OS
   // installation carries build output, not a build tree, so resolving CMake
@@ -1483,6 +1566,29 @@ export function createCppBom(path, options) {
         boundaries: new Map(),
       }
     : resolveCmakeContext(path, options);
+  // How the project is built: its presets and compilers (formulation), its
+  // hardening options (project properties) and its own include directories.
+  const isImageScan = hasAnyProjectType(
+    ["docker", "oci", "container", "os"],
+    options,
+    false,
+  );
+  // Code carried in the tree under a license of its own, outside the sources
+  // of the dependencies CMake already knows and the build trees
+  const vendored = isImageScan
+    ? { components: [], dirs: [] }
+    : findVendoredCode(path, options, [
+        ...[...(cmakeContext.boundaries || new Map())]
+          .filter(([, e]) => e?.kind === "fetch" || e?.kind === "submodule")
+          .map(([dir]) => resolve(path, dir)),
+        ...cmakeBuildDirCandidates(path, options),
+      ]);
+  if (vendored.components.length) {
+    pkgList = pkgList.concat(vendored.components);
+  }
+  const buildContext = isImageScan
+    ? emptyCppBuildContext()
+    : resolveCppBuildContext(path, options, cmakeContext, vendored.dirs);
   if (cmakeContext.rootProject?.version && parentComponent) {
     const resolvedVersion = cmakeContext.rootProject.version;
     const scrapedVersion = parentComponent.version;
@@ -1554,6 +1660,7 @@ export function createCppBom(path, options) {
         { name: "cdx:cmake:resolvedVia", value: "gitclone-script" },
       ],
     };
+    pkgList = withoutScrapedFetchDependency(pkgList, dep.name);
     if (dep.sourceDir) {
       depComponent.properties.push({
         name: "cdx:cmake:sourceDir",
@@ -1611,30 +1718,44 @@ export function createCppBom(path, options) {
     // We collect OS packages with the word dev in the name using osquery here
     // rpm, deb and ebuild are supported
     // TODO: For archlinux and alpine users we need a different mechanism to collect this information
-    for (const queryCategory of Object.keys(cosDbQueries)) {
-      const queryObj = cosDbQueries[queryCategory];
-      const results = executeOsQuery(queryObj.query);
-      const dlist = convertOSQueryResults(
-        queryCategory,
-        queryObj,
-        results,
-        true,
-        {
-          deb: executeDpkgList,
-          rpm: executeRpmList,
-          apk: executeApkList,
-          ebuild: executeEqueryList,
-          alpm: executeAlpmList,
-        },
-      );
-      if (dlist?.length) {
-        osPkgsList = osPkgsList.concat(dlist);
+    // The packages only resolve the headers named by the usages slice, so
+    // they are not queried when there is neither a slice to reuse nor a C or
+    // C++ source for atom to read.
+    if (
+      (options.usagesSlicesFile && safeExistsSync(options.usagesSlicesFile)) ||
+      hasCppSources(path, options)
+    ) {
+      for (const queryCategory of Object.keys(cosDbQueries)) {
+        const queryObj = cosDbQueries[queryCategory];
+        const results = executeOsQuery(queryObj.query);
+        const dlist = convertOSQueryResults(
+          queryCategory,
+          queryObj,
+          results,
+          true,
+          {
+            deb: executeDpkgList,
+            rpm: executeRpmList,
+            apk: executeApkList,
+            ebuild: executeEqueryList,
+            alpm: executeAlpmList,
+          },
+        );
+        if (dlist?.length) {
+          osPkgsList = osPkgsList.concat(dlist);
+        }
       }
     }
     // Now we check with atom and attempt to detect all external modules via usages
     // We pass the current list of packages so that we enhance the current list and replace
     // components inadvertently. For example, we might resolved a name, version and url information already via cmake
-    const retMap = getCppModules(path, options, osPkgsList, pkgList);
+    const retMap = getCppModules(
+      path,
+      options,
+      osPkgsList,
+      pkgList,
+      buildContext.isFirstPartyHeader,
+    );
     if (retMap.pkgList?.length) {
       pkgList = pkgList.concat(retMap.pkgList);
     }
@@ -1678,6 +1799,33 @@ export function createCppBom(path, options) {
     ]);
   }
 
+  if (subProjects.length) {
+    if (parentComponent) {
+      parentComponent.components = parentComponent.components || [];
+      for (const sub of subProjects) {
+        if (
+          !parentComponent.components.some(
+            (c) => c["bom-ref"] === sub["bom-ref"],
+          ) &&
+          sub["bom-ref"] !== parentComponent["bom-ref"]
+        ) {
+          parentComponent.components.push(sub);
+        }
+      }
+    } else {
+      // with no project at the scan root there is nothing to nest them under
+      pkgList = pkgList.concat(subProjects);
+    }
+  }
+  if (parentComponent && buildContext.parentProperties.length) {
+    parentComponent.properties = (parentComponent.properties || []).filter(
+      (p) =>
+        !buildContext.parentProperties.some(
+          (bp) => bp.name === p.name && bp.value === p.value,
+        ),
+    );
+    parentComponent.properties.push(...buildContext.parentProperties);
+  }
   if (parentComponent) {
     sanitizeUnresolvedPurl(parentComponent);
     if (parentComponent.components) {
@@ -1694,6 +1842,7 @@ export function createCppBom(path, options) {
     src: path,
     parentComponent,
     dependencies,
+    formulationList: buildContext.formulationComponents,
   });
 }
 
@@ -2024,6 +2173,50 @@ export async function createSwiftBom(path, options) {
   let parentComponent = {};
   const completedPath = [];
   let packageArgsMessageShown = false;
+  // Packages that only the root package's tests use are scoped optional so
+  // that `--required-only` can narrow the BOM. A package some project (or
+  // some production path) needs stays required.
+  const swiftScopes = {
+    optionalRefs: new Set(),
+    requiredRefs: new Set(),
+    optionalNames: new Set(),
+    requiredNames: new Set(),
+  };
+  const classifiedPaths = new Set();
+  const classifyScopes = (basePath, swiftCommand, treeData) => {
+    if (classifiedPaths.has(basePath)) {
+      return;
+    }
+    classifiedPaths.add(basePath);
+    const dumpArgs =
+      swiftCommand === "xcrun"
+        ? ["swift", "package", "dump-package"]
+        : ["package", "dump-package"];
+    const dumpResult = safeSpawnSync(swiftCommand, dumpArgs, {
+      cwd: basePath,
+    });
+    if (!dumpResult?.stdout) {
+      return;
+    }
+    let dumpJson;
+    let treeJson;
+    try {
+      dumpJson = JSON.parse(Buffer.from(dumpResult.stdout).toString());
+    } catch (_e) {
+      return;
+    }
+    try {
+      treeJson = treeData ? JSON.parse(treeData) : undefined;
+    } catch (_e) {
+      treeJson = undefined;
+    }
+    const scopes = classifySwiftDependencyScopes(dumpJson, treeJson);
+    for (const key of Object.keys(swiftScopes)) {
+      for (const value of scopes[key]) {
+        swiftScopes[key].add(value);
+      }
+    }
+  };
   if (pkgResolvedFiles.length) {
     for (const f of pkgResolvedFiles) {
       if (!parentComponent || !Object.keys(parentComponent).length) {
@@ -2147,6 +2340,33 @@ export async function createSwiftBom(path, options) {
           detail: "the swift package command failed",
         });
       }
+      // Scope classification also works without the dependency graph
+      classifyScopes(basePath, swiftCommand, treeData);
+    }
+  }
+  // Scope packages that only tests use as optional. A package some project or
+  // production path needs keeps its default (required) scope; evidence-based
+  // filtering after generation can still promote an optional package when
+  // the sources are found to use it.
+  for (const apkg of pkgList) {
+    if (apkg?.scope || !apkg?.name) {
+      continue;
+    }
+    const refs = [apkg["bom-ref"], apkg.purl].filter(Boolean);
+    const names = [String(apkg.name).toLowerCase()];
+    for (const aproperty of apkg.properties || []) {
+      if (aproperty?.name === "cdx:swift:packageName" && aproperty.value) {
+        names.push(String(aproperty.value).toLowerCase());
+      }
+    }
+    const isOptional =
+      refs.some((r) => swiftScopes.optionalRefs.has(r)) ||
+      names.some((n) => swiftScopes.optionalNames.has(n));
+    const isRequired =
+      refs.some((r) => swiftScopes.requiredRefs.has(r)) ||
+      names.some((n) => swiftScopes.requiredNames.has(n));
+    if (isOptional && !isRequired) {
+      apkg.scope = "optional";
     }
   }
   if (shouldFetchLicense()) {
@@ -2217,7 +2437,7 @@ export async function createCocoaBom(path, options) {
       "application",
     );
     const podfileLock = loadYaml(readFileSync(lockFile, "utf-8"));
-    const pods = await parsePodfileLock(podfileLock, projectPath);
+    const pods = await parsePodfileLock(podfileLock, projectPath, path);
     const allObjects = new Map();
     if (options) {
       // Every remote podspec the scan below reads is known from the lockfile,

package/lib/core/activity.js

@@ -285,6 +285,18 @@ const MANIFEST_ACTIVITY_HINTS = new Map([
   ],
 ]);
 const SENSITIVE_CONFIG_ACTIVITY_HINTS = [
+  {
+    matcher: (_lowerPath, baseName) =>
+      baseName === ".terraformrc" ||
+      baseName === "terraform.rc" ||
+      baseName === "credentials.tfrc.json",
+    metadata: {
+      classification: "credential",
+      ecosystem: "terraform",
+      label: "Terraform CLI configuration",
+      sensitive: true,
+    },
+  },
   {
     matcher: (lowerPath, _baseName) =>
       lowerPath.includes("/.cargo/config.toml") ||

package/lib/core/env.js

@@ -111,19 +111,27 @@ export function shouldFetchLicense() {
   );
 }
 
+/**
+ * Determines whether registry provenance, such as publish times, publishers
+ * and trusted-publishing signals, is wanted. `--bom-audit` turns it on through
+ * CDXGEN_FETCH_PKG_METADATA. Local data cannot answer these questions, so a
+ * package whose licence and description are known locally is still looked up.
+ *
+ * @returns {boolean} True if CDXGEN_FETCH_PKG_METADATA is "true" or "1"
+ */
+export function shouldFetchRegistryProvenance() {
+  return ["true", "1"].includes(
+    readEnvironmentVariable("CDXGEN_FETCH_PKG_METADATA"),
+  );
+}
+
 /**
  * Determines whether remote package metadata should be fetched for enrichment.
  *
  * @returns {boolean} True when registry metadata enrichment is enabled.
  */
 export function shouldFetchPackageMetadata() {
-  return (
-    shouldFetchLicense() ||
-    (readEnvironmentVariable("CDXGEN_FETCH_PKG_METADATA") &&
-      ["true", "1"].includes(
-        readEnvironmentVariable("CDXGEN_FETCH_PKG_METADATA"),
-      ))
-  );
+  return shouldFetchLicense() || shouldFetchRegistryProvenance();
 }
 
 /**
@@ -457,7 +465,7 @@ export const PROJECT_TYPE_ALIASES = {
   perl: ["perl", "cpan", "carton", "cpanfile"],
   spack: ["spack"],
   terraform: ["terraform", "opentofu", "tf"],
-  c: ["c", "cpp", "c++", "conan", "collider"],
+  c: ["c", "cpp", "c++", "conan", "collider", "cmake", "meson", "vcpkg"],
   clojure: ["clojure", "edn", "clj", "leiningen"],
   github: ["github", "actions"],
   hbom: ["hbom", "hardware"],
@@ -491,9 +499,9 @@ export const PROJECT_TYPE_ALIASES = {
     "visionos",
   ],
   binary: ["binary", "blint"],
-  oci: ["docker", "oci", "container", "podman", "rootfs", "oci-dir"],
+  oci: ["docker", "image", "oci", "container", "podman", "rootfs", "oci-dir"],
   cocoa: ["cocoa", "cocoapods", "objective-c", "swift", "ios"],
-  scala: ["scala", "scala3", "sbt", "mill"],
+  scala: ["scala", "scala3", "sbt", "mill", "scala-cli"],
   nix: ["nix", "nixos", "flake"],
   zig: ["zig", "zon"],
   gleam: ["gleam"],
@@ -643,11 +651,10 @@ export function hasAnyProjectType(projectTypes, options, defaultStatus = true) {
   const allProjectTypes = [...projectTypes];
   // Convert the project types into base types
   const baseProjectTypes = [];
-  // Support for arbitrary versioned ruby type
-  if (
-    options.projectType?.length &&
-    projectTypes.filter((p) => p.startsWith("ruby")).length
-  ) {
+  // Support for arbitrary versioned ruby type: a requested ruby3.4.1 is a
+  // ruby project. The test is on the requested types; testing the queried
+  // ones made every scan a ruby scan whenever "ruby" was asked about.
+  if (options.projectType?.some((p) => p.startsWith("ruby"))) {
     baseProjectTypes.push("ruby");
   }
   // Versioned JVM build tool types such as maven3.9.9 or gradle8.14 are java