@cdxgen/cdxgen 13.3.0 → 13.4.0

package/lib/ecosystems/parsers-js.js

@@ -31,6 +31,7 @@ import {
   loadNpmrcConfig,
   normalizePnpmLockKey,
   parsePnpmGitLockKey,
+  resolveNpmLicense,
   setNpmDevelopmentProperty,
   setNpmOptionalProperty,
   setNpmPeerProperty,
@@ -135,6 +136,11 @@ const NPM_LIFECYCLE_JS_SOURCE_FILE_PATTERN = /\.[cm]?[jt]sx?$/i;
 
 const SHELL_ENV_ASSIGNMENT_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*=.*/;
 
+// The root component's name when neither package.json, --project-name, nor
+// the directory names one (a tree rooted at a file-system root): a purl needs
+// a name.
+const UNNAMED_NPM_PROJECT = "unnamed-project";
+
 function splitLifecycleScriptCommands(scriptValue) {
   const commands = [];
   let current = "";
@@ -432,6 +438,9 @@ function analyzeNpmLifecycleScripts(pkgJsonFile, scripts) {
  *   identity evidence and skip npm registry metadata fetching.
  * @param {boolean} [securityProps=false] When true, append npm security-relevant
  *   properties such as lifecycle scripts, binaries, native addons, and deprecation notices.
+ * @param {Object} [options] Options
+ * @param {boolean} [options.fetchMetadata=true] When false, skip the npm
+ *   registry round so that a caller can make one round for many manifests.
  * @returns {Promise<Array<object>>} Parsed component list, enriched with npm
  *   registry metadata when available and not in simple mode.
  */
@@ -439,11 +448,15 @@ export async function parsePkgJson(
   pkgJsonFile,
   simple = false,
   securityProps = false,
+  options = {},
 ) {
   const pkgList = [];
   if (safeExistsSync(pkgJsonFile)) {
     try {
-      const pkgData = JSON.parse(readFileSync(pkgJsonFile, "utf8"));
+      // npm and Node.js both accept a package.json that starts with a UTF-8 BOM
+      const pkgData = JSON.parse(
+        readFileSync(pkgJsonFile, "utf8").replace(/^\uFEFF/, ""),
+      );
       const pkgIdentifier = parsePackageJsonName(pkgData.name);
       let name = pkgIdentifier.fullName || pkgData.name;
       if (!name && !pkgJsonFile.includes("node_modules")) {
@@ -471,7 +484,7 @@ export async function parsePkgJson(
         purl: purl,
         "bom-ref": decodeURIComponent(purl),
         author: authorString,
-        license: pkgData.license,
+        license: resolveNpmLicense(pkgData),
       };
       if (pkgData.homepage) {
         apkg.homepage = { url: pkgData.homepage };
@@ -662,7 +675,12 @@ export async function parsePkgJson(
       // continue regardless of error
     }
   }
-  if (!simple && shouldFetchPackageMetadata() && pkgList?.length) {
+  if (
+    !simple &&
+    options.fetchMetadata !== false &&
+    shouldFetchPackageMetadata() &&
+    pkgList?.length
+  ) {
     if (DEBUG_MODE) {
       console.log(
         `About to fetch npm registry metadata for ${pkgList.length} packages in parsePkgJson`,
@@ -899,6 +917,9 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
 
     let pkg;
     let purlString;
+    // The license the installed package.json declares, for a package whose
+    // lockfile entry carries none (see the spoofing check below).
+    let installedManifestLicense;
     const author = nodePackage.author;
     const authorString =
       author instanceof Object
@@ -908,8 +929,18 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
         : author;
     if (node === rootNode) {
       const projectGroup = options.projectGroup;
+      // A package.json without a name still installs, so the root of a deep
+      // tree can lack one: name the root component after its directory, the
+      // way the scans without node_modules already do, rather than fail to
+      // build its purl (issue 4440). A tree rooted at a file-system root has
+      // no directory name either, and neither has the --project-name the CLI
+      // derives from it.
       const projectName =
-        "project-name" in options ? options.projectName : node.packageName;
+        ("project-name" in options && options.projectName) ||
+        node.packageName ||
+        basename(node.path || "") ||
+        node.name ||
+        UNNAMED_NPM_PROJECT;
       const projectVersion = options.projectVersion || node.version;
       // A package.json need not declare a version, and npm installs without
       // one, so the root of a lockfile can reach here with none. A purl carries
@@ -927,8 +958,7 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
       pkg = {
         author: authorString,
         group: options.projectGroup || "",
-        name:
-          "project-name" in options ? options.projectName : node.packageName,
+        name: projectName,
         version: options.projectVersion || node.version,
         type: "application",
         purl: purlString,
@@ -964,13 +994,19 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
         });
       }
     } else {
+      // A package.json without a name installs as a file: or workspace
+      // dependency too, and arborist then has no packageName for it: such a
+      // package goes by the name its folder has in the tree, which is also
+      // the name the edges into it carry (issue 4440).
+      const packageName =
+        node.packageName || node.name || basename(node.path || "");
       let namespace = "";
-      let name = node.packageName;
-      if (node.packageName.startsWith("@")) {
-        const slashIndex = node.packageName.indexOf("/");
+      let name = packageName;
+      if (packageName.startsWith("@")) {
+        const slashIndex = packageName.indexOf("/");
         if (slashIndex > 0) {
-          namespace = node.packageName.substring(0, slashIndex);
-          name = node.packageName.substring(slashIndex + 1);
+          namespace = packageName.substring(0, slashIndex);
+          name = packageName.substring(slashIndex + 1);
         }
       }
       let qualifiers = null;
@@ -1083,9 +1119,17 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
         });
       }
       if (node?.binPaths?.length) {
+        // Absolute bin paths would leak the generating machine's home
+        // directory into the SBOM and differ between checkouts, so they are
+        // emitted relative to the project root the package was resolved from,
+        // with forward slashes on every platform (issue 4467).
         pkg.properties.push({
           name: "cdx:npm:binPaths",
-          value: node.binPaths.join(", "),
+          value: node.binPaths
+            .map((binPath) =>
+              relative(srcFilePath, binPath).split(_sep).join("/"),
+            )
+            .join(", "),
         });
       }
       if (nodePackage?.bin) {
@@ -1154,9 +1198,14 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
         try {
           const onDiskPkg =
             diskPkg || JSON.parse(readFileSync(packageJsonPath, "utf8"));
-          if (!onDiskPkg.name || onDiskPkg.name !== node.packageName) {
+          // A package that declares no name on disk and none in the lockfile
+          // (a nameless file: dependency) disagrees with nothing.
+          if (
+            (onDiskPkg.name || node.packageName) &&
+            onDiskPkg.name !== node.packageName
+          ) {
             console.warn(
-              `\x1b[1;35mWARNING: Package name spoofing detected for ${node.packageName}! Lockfile says ${node.packageName}, but disk says ${onDiskPkg.name}.\x1b[0m`,
+              `\x1b[1;35mWARNING: Package name spoofing detected for ${packageName}! Lockfile says ${node.packageName}, but disk says ${onDiskPkg.name}.\x1b[0m`,
             );
             if (onDiskPkg.name) {
               pkg.properties.push({
@@ -1165,9 +1214,12 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
               });
             }
           }
-          if (!onDiskPkg.version || onDiskPkg.version !== node.version) {
+          if (
+            (onDiskPkg.version || node.version) &&
+            onDiskPkg.version !== node.version
+          ) {
             console.warn(
-              `\x1b[1;35mWARNING: Package version spoofing detected for ${node.packageName}! Lockfile says ${node.version}, but disk says ${onDiskPkg.version}.\x1b[0m`,
+              `\x1b[1;35mWARNING: Package version spoofing detected for ${packageName}! Lockfile says ${node.version}, but disk says ${onDiskPkg.version}.\x1b[0m`,
             );
             if (onDiskPkg.version) {
               pkg.properties.push({
@@ -1176,6 +1228,17 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
               });
             }
           }
+          // npm records only the modern `license` field in the lockfile, so a
+          // package that declares the legacy `licenses` array (issue 4466) has
+          // no license in the tree loaded from it. The installed manifest is
+          // then the only local declaration, and only when it is the package
+          // the lockfile describes.
+          if (
+            onDiskPkg.name === node.packageName &&
+            onDiskPkg.version === node.version
+          ) {
+            installedManifestLicense = resolveNpmLicense(onDiskPkg);
+          }
         } catch (_err) {
           // ignore
         }
@@ -1231,9 +1294,11 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
         pkg.description = nodePackage.description;
       }
     }
-    if (nodePackage?.license) {
+    const npmLicense =
+      resolveNpmLicense(nodePackage) || installedManifestLicense;
+    if (npmLicense) {
       // License will be overridden if shouldFetchLicense() is enabled
-      pkg.license = nodePackage.license;
+      pkg.license = npmLicense;
     }
     const deprecatedMessage = nodePackage?.deprecated;
     if (deprecatedMessage) {
@@ -1607,7 +1672,7 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
         `About to fetch npm registry metadata for ${pkgList.length} packages in parsePkgLock`,
       );
     }
-    pkgList = await getNpmMetadata(pkgList);
+    pkgList = await getNpmMetadata(pkgList, undefined, { npmrcConfig });
     return { pkgList, dependenciesList };
   }
   return {
@@ -2457,7 +2522,7 @@ export async function parseYarnLock(
         `About to fetch npm registry metadata for ${pkgList.length} packages in parseYarnLock`,
       );
     }
-    pkgList = await getNpmMetadata(pkgList);
+    pkgList = await getNpmMetadata(pkgList, undefined, { npmrcConfig });
     return {
       pkgList,
       dependenciesList,
@@ -2476,6 +2541,7 @@ export async function parseYarnLock(
  */
 export async function parseNodeShrinkwrap(swFile) {
   const pkgList = [];
+  const npmrcConfig = loadNpmrcConfig(dirname(swFile));
   if (safeExistsSync(swFile)) {
     const lockData = JSON.parse(readFileSync(swFile, "utf8"));
     const pkgKeys = Object.keys(lockData);
@@ -2534,7 +2600,7 @@ export async function parseNodeShrinkwrap(swFile) {
         `About to fetch npm registry metadata for ${pkgList.length} packages in parseNodeShrinkwrap`,
       );
     }
-    return await getNpmMetadata(pkgList);
+    return await getNpmMetadata(pkgList, undefined, { npmrcConfig });
   }
   return pkgList;
 }
@@ -2702,13 +2768,13 @@ export async function getVersionNumPnpm(depPkg, relativePath) {
       : join(version.replaceAll("../", "packages/"), "package.json");
     if (safeExistsSync(relativePkgJson)) {
       const importedComponentObj = await parsePkgJson(relativePkgJson, true);
-      version = importedComponentObj[0].version;
+      version = importedComponentObj[0]?.version;
     } else if (safeExistsSync(join(version, "package.json"))) {
       const importedComponentObj = await parsePkgJson(
         join(version, "package.json"),
         true,
       );
-      version = importedComponentObj[0].version;
+      version = importedComponentObj[0]?.version;
     } else if (
       safeExistsSync(join(version.replaceAll("../", ""), "package.json"))
     ) {
@@ -2716,7 +2782,7 @@ export async function getVersionNumPnpm(depPkg, relativePath) {
         join(version.replaceAll("../", ""), "package.json"),
         true,
       );
-      version = importedComponentObj[0].version;
+      version = importedComponentObj[0]?.version;
     }
   } else {
     // version: 3.0.1(ajv@8.14.0) or, under lockfile 5, 3.0.1_ajv@8.14.0
@@ -4125,7 +4191,7 @@ export async function parsePnpmLock(
         `About to fetch npm registry metadata for ${pkgList.length} packages in parsePnpmLock`,
       );
     }
-    pkgList = await getNpmMetadata(pkgList);
+    pkgList = await getNpmMetadata(pkgList, undefined, { npmrcConfig });
     return {
       pkgList,
       dependenciesList,
@@ -4155,7 +4221,7 @@ export async function parseBowerJson(bowerJsonFile) {
         group: pkgIdentifier.scope || "",
         version: pkgData.version || "",
         description: pkgData.description || "",
-        license: pkgData.license || "",
+        license: resolveNpmLicense(pkgData) || "",
         properties: [
           {
             name: "internal:SrcFile",

package/lib/ecosystems/parsers-julia.js

@@ -1,7 +1,6 @@
 import { readFileSync } from "node:fs";
 
-import * as toml from "smol-toml";
-
+import { parseToml } from "../core/toml.js";
 import { tryBuildPurl } from "../inventory/purl.js";
 
 /**
@@ -260,7 +259,7 @@ const JULIA_STDLIBS = new Set([
  */
 function readTomlFile(filePath) {
   try {
-    return toml.parse(readFileSync(filePath, "utf-8"));
+    return parseToml(readFileSync(filePath, "utf-8"));
   } catch (error) {
     console.warn(`Failed to parse ${filePath}: ${error.message}`);
     return undefined;

package/lib/ecosystems/parsers-jvm.js

@@ -1,14 +1,23 @@
-import { readFileSync } from "node:fs";
-import { basename, dirname, resolve } from "node:path";
+import { readdirSync, readFileSync } from "node:fs";
+import {
+  basename,
+  dirname,
+  isAbsolute,
+  join,
+  relative,
+  resolve,
+} from "node:path";
 
 import { build } from "@cdxgen/cdx-purl";
 import { parseEDNString } from "edn-data";
 
-import { readEnvironmentVariable } from "../core/activity.js";
+import { DEBUG_MODE, readEnvironmentVariable } from "../core/activity.js";
 import { recordDegradation } from "../core/buildLedger.js";
 import { includeMavenTestScope } from "../core/env.js";
+import { safeExistsSync } from "../core/fs.js";
 import { thoughtLog } from "../core/logger.js";
 import { applyPurl } from "../inventory/purl.js";
+import { parseScalaArtifact } from "../inventory/scalaCoords.js";
 import { xml2js } from "../parsers/xml.js";
 
 /**
@@ -173,6 +182,158 @@ export function parsePom(pomFile) {
   return { isQuarkus, pomPurl, modules, properties, dependencies: deps };
 }
 
+/** Bound on the reactor walk. A pom's module list is untrusted input, so the
+ * walk cannot trust it to stay finite. */
+const REACTOR_MODULE_LIMIT = 500;
+
+/**
+ * Every pom file of the reactor an aggregator pom builds: its own modules,
+ * recursively through nested aggregators. A recursive dependency:tree run
+ * from the aggregator resolves all of them, so their own runs repeat it.
+ *
+ * A module entry names a directory relative to its pom, sometimes a direct
+ * path to a pom file. Entries that are absolute, or that climb out of the
+ * aggregator's directory, are ignored: the scan never listed them, and the
+ * walk must not follow a pom's pointers outside its own tree.
+ *
+ * @param {string} rootPom Path of the aggregator pom.
+ * @returns {Set<string>} Pom files of the reactor, excluding the aggregator.
+ */
+export function resolveReactorPomFiles(rootPom) {
+  const pomFiles = new Set();
+  const visited = new Set();
+  const rootDir = dirname(resolve(rootPom));
+  const queue = [resolve(rootPom)];
+  while (queue.length > 0 && pomFiles.size < REACTOR_MODULE_LIMIT) {
+    const pom = queue.shift();
+    if (visited.has(pom)) {
+      continue;
+    }
+    visited.add(pom);
+    let modules;
+    try {
+      modules = parsePom(pom)?.modules;
+    } catch (_err) {
+      // An unreadable module pom ends this branch of the walk.
+      continue;
+    }
+    for (const module of modules || []) {
+      if (typeof module !== "string") {
+        continue;
+      }
+      const cleaned = module.trim().replaceAll("\\", "/");
+      if (
+        !cleaned ||
+        cleaned.includes("\0") ||
+        cleaned.startsWith("/") ||
+        /^[A-Za-z]:/.test(cleaned)
+      ) {
+        continue;
+      }
+      const modulePom = resolve(
+        dirname(pom),
+        cleaned.endsWith(".xml") ? cleaned : join(cleaned, "pom.xml"),
+      );
+      const withinRoot = relative(rootDir, modulePom);
+      if (
+        !withinRoot ||
+        withinRoot.startsWith("..") ||
+        isAbsolute(withinRoot) ||
+        !safeExistsSync(modulePom)
+      ) {
+        continue;
+      }
+      if (visited.has(modulePom)) {
+        continue;
+      }
+      pomFiles.add(modulePom);
+      queue.push(modulePom);
+    }
+  }
+  return pomFiles;
+}
+
+/**
+ * File name stem for the trees of one recursive dependency:tree run. Maven
+ * expands the project expressions for each reactor module, so every module
+ * writes its own file; a fixed name would hold only the last module's tree.
+ */
+export const MAVEN_MODULE_TREE_STEM =
+  // biome-ignore lint/suspicious/noTemplateCurlyInString: Maven expands these project expressions itself
+  "${project.groupId}-${project.artifactId}";
+
+/**
+ * The groupId-artifactId key a module's tree file is named by.
+ *
+ * @param {string} pomFile Path of the module pom.
+ * @returns {string|undefined} The key, or undefined when the pom cannot be read.
+ */
+function moduleTreeKey(pomFile) {
+  try {
+    const { groupId, artifactId } = parsePom(pomFile)?.properties || {};
+    return groupId && artifactId ? `${groupId}-${artifactId}` : undefined;
+  } catch (_err) {
+    return undefined;
+  }
+}
+
+/**
+ * Read the trees one dependency:tree run wrote, one file per reactor module
+ * (see MAVEN_MODULE_TREE_STEM), and tell which of the reactor's pom files
+ * they cover. The pom the run started from comes first, then its modules in
+ * reactor order. A module pom counts as covered only when its own tree is
+ * among the files, so a module the run left out still runs on its own.
+ *
+ * @param {string} treeDir Directory the run wrote its trees into.
+ * @param {string} rootPom The pom file the run started from.
+ * @returns {{trees: Array<{pomFile: string, text: string}>, covered: Set<string>}}
+ */
+export function readMavenModuleTrees(treeDir, rootPom) {
+  const trees = [];
+  const covered = new Set();
+  let names;
+  try {
+    names = readdirSync(treeDir).sort();
+  } catch (_err) {
+    return { trees, covered };
+  }
+  const byKey = new Map();
+  for (const name of names) {
+    const stem = name.replace(/\.(json|txt)$/, "");
+    if (stem !== name && !byKey.has(stem)) {
+      byKey.set(stem, name);
+    }
+  }
+  const take = (key, pomFile) => {
+    const name = byKey.get(key);
+    if (!name) {
+      return false;
+    }
+    byKey.delete(key);
+    trees.push({
+      pomFile,
+      text: readFileSync(join(treeDir, name), { encoding: "utf-8" }),
+    });
+    return true;
+  };
+  const rootKey = moduleTreeKey(rootPom);
+  if (rootKey) {
+    take(rootKey, rootPom);
+  }
+  for (const modulePom of resolveReactorPomFiles(rootPom)) {
+    const key = moduleTreeKey(modulePom);
+    if (key && take(key, modulePom)) {
+      covered.add(modulePom);
+    }
+  }
+  // Trees no pom of the reactor names, such as the single file of a Maven
+  // that did not expand the expressions, are kept under the root pom.
+  for (const key of [...byKey.keys()]) {
+    take(key, rootPom);
+  }
+  return { trees, covered };
+}
+
 function mapMavenScope(componentScope, isOptional = false) {
   if (isOptional || componentScope === "test") {
     return "optional";
@@ -471,10 +632,16 @@ export function parseMavenTree(rawOutput, pomFile) {
 /**
  * Parse mill dependencies from file
  *
+ * Scala artifacts are emitted in the sbt purl form: the Scala binary suffix
+ * is stripped from the artifact name and the version it carried is recorded
+ * in the cdx:scala:compilerVersion property, so one library keeps a single
+ * purl whichever build tool reported it.
+ *
  * @param {string} module name of the module
  * @param {map} dependencies the parsed dependencies
  * @param {map} relations a map containing all relations
  * @param {string} millRootPath root of the project
+ * @param {string} logFileName name of the tree log the module wrote
  *
  * @returns the bom-ref of the module
  */
@@ -483,6 +650,7 @@ export function parseMillDependency(
   dependencies,
   relations,
   millRootPath,
+  logFileName = "ivyDepsTree.log",
 ) {
   const treeRegex = /^(?<treeIndentation>(?:[?├│└─ ]{3})+)(?<dependency>.*)/m;
   const ESC = "\\x1B";
@@ -522,12 +690,7 @@ export function parseMillDependency(
     // Now put the versions we removed above back into the path and replace the brackets at the same time
     moduleFilePath = moduleFilePath.replace("[]", `/${versionNumber}/`);
   }
-  moduleFilePath = resolve(
-    millRootPath,
-    "out",
-    moduleFilePath,
-    "ivyDepsTree.log",
-  );
+  moduleFilePath = resolve(millRootPath, "out", moduleFilePath, logFileName);
   moduleComponent.properties = [
     {
       name: "internal:SrcFile",
@@ -547,7 +710,27 @@ export function parseMillDependency(
       ],
     },
   };
-  const dependencyTreeLog = readFileSync(moduleFilePath, "utf-8");
+  let dependencyTreeLog;
+  try {
+    dependencyTreeLog = readFileSync(moduleFilePath, "utf-8");
+  } catch (_err) {
+    // A cross-published module writes its log under the dotted version
+    // directory (0.23.38) while the resolve output prints it with
+    // underscores (0_23_38)
+    try {
+      dependencyTreeLog = readFileSync(
+        moduleFilePath.replace(/(\d+)_(\d+)_(\d+)/, "$1.$2.$3"),
+        "utf-8",
+      );
+    } catch (_err2) {
+      if (DEBUG_MODE) {
+        console.log(
+          `No Mill dependency tree log was written for the module ${module}.`,
+        );
+      }
+      return moduleComponent["bom-ref"];
+    }
+  }
   const dependencyTreeLines = dependencyTreeLog
     .trim()
     .split("\n")
@@ -568,11 +751,20 @@ export function parseMillDependency(
       [group, name, version] = match.groups.dependency.split(":");
       version = versionRegex.exec(version).groups.version;
     }
+    const scalaInfo = parseScalaArtifact(name);
     const component = completeComponent({
       group,
-      name,
+      name: scalaInfo.purlName,
       version,
     });
+    if (scalaInfo.binaryVersion) {
+      component.properties = [
+        {
+          name: "cdx:scala:compilerVersion",
+          value: scalaInfo.binaryVersion,
+        },
+      ];
+    }
     if (!dependencies.has(component["bom-ref"])) {
       dependencies.set(component["bom-ref"], component);
       relations.set(component["bom-ref"], []);
@@ -587,7 +779,7 @@ export function parseMillDependency(
   return moduleComponent["bom-ref"];
 }
 
-function completeComponent(component) {
+export function completeComponent(component) {
   component["type"] = component.group ? "library" : "application";
   try {
     const purl = build({