@cdxgen/cdxgen 13.2.0 → 13.4.0

package/lib/ecosystems/parsers-python.js

@@ -4,13 +4,13 @@ import process from "node:process";
 import { URL } from "node:url";
 
 import { build } from "@cdxgen/cdx-purl";
-import * as toml from "smol-toml";
 import { parse as _load } from "yaml";
 
 import { DEBUG_MODE } from "../core/activity.js";
 import { recordDegradation } from "../core/buildLedger.js";
 import { getAllFiles, safeExistsSync, safeSpawnSync } from "../core/fs.js";
 import { PYTHON_STD_MODULES } from "../core/state.js";
+import { parseToml } from "../core/toml.js";
 import { tryBuildPurl } from "../inventory/purl.js";
 import {
   createExternalReferenceKey,
@@ -95,8 +95,11 @@ export function parsePyRequiresDist(dist_string) {
  * Method to parse pipfile.lock data
  *
  * @param {Object} lockData JSON data from Pipfile.lock
+ * @param {Object[]} [installedMetadata] Components parsed from the installed
+ *   `*.dist-info/METADATA` files of this machine, which answer before PyPI
+ *   is asked
  */
-export async function parsePiplockData(lockData) {
+export async function parsePiplockData(lockData, installedMetadata) {
   const pkgList = [];
   Object.keys(lockData)
     .filter((i) => i !== "_meta")
@@ -105,11 +108,16 @@ export async function parsePiplockData(lockData) {
       Object.keys(depBlock).forEach((p) => {
         const pkg = depBlock[p];
         if (Object.hasOwn(pkg, "version")) {
-          const versionStr = pkg.version.replace("==", "");
+          // `===` (arbitrary equality) pins as tightly as `==`, so both are
+          // stripped whole rather than leaving a stray `=` behind.
+          const versionStr = pkg.version.replace(/^\s*={2,3}\s*/, "");
           pkgList.push({ name: p, version: versionStr });
         }
       });
     });
+  // The installed distributions answer first, so PyPI is asked only about
+  // the packages they do not cover.
+  enrichFromInstalledMetadata(pkgList, installedMetadata);
   return await getPyMetadata(pkgList, false);
 }
 
@@ -272,8 +280,12 @@ function collectPythonManifestSource(pkg) {
     { kind: "url", value: pkg?.vcs?.url },
     { kind: "url", value: pkg?.source?.url },
     { kind: "path", value: pkg?.source?.path },
+    { kind: "path", value: pkg?.source?.directory },
     { kind: "path", value: pkg?.source?.editable },
     { kind: "path", value: pkg?.source?.virtual },
+    { kind: "path", value: pkg?.directory?.path },
+    { kind: "url", value: pkg?.archive?.url },
+    { kind: "path", value: pkg?.archive?.path },
   ];
   for (const candidate of sourceCandidates) {
     if (typeof candidate.value !== "string" || !candidate.value.trim()) {
@@ -393,7 +405,7 @@ export function parsePyProjectTomlFile(tomlFile) {
   const groupDepsKeys = {};
   const dependencySourceMap = {};
   try {
-    tomlData = toml.parse(readFileSync(tomlFile, { encoding: "utf-8" }));
+    tomlData = parseToml(readFileSync(tomlFile, { encoding: "utf-8" }));
   } catch (err) {
     console.log(`Error while parsing the pyproject file ${tomlFile}.`, err);
   }
@@ -786,14 +798,114 @@ function collectPythonLockMetadataDistributionReferences(fileEntries) {
   return distributionReferences;
 }
 
+/**
+ * Normalise a Python distribution name the way PEP 503 does, so an installed
+ * distribution and the lockfile entry it belongs to compare equal whatever
+ * separator and case each of them used.
+ *
+ * @param {string} value Distribution name
+ * @returns {string|undefined} The normalised name, or undefined for no name
+ */
+export function normalizePep503Name(value) {
+  if (typeof value !== "string" || !value.trim()) {
+    return undefined;
+  }
+  return value
+    .trim()
+    .toLowerCase()
+    .replaceAll(/[-_.]+/g, "-");
+}
+
+/**
+ * The key an installed distribution and a lockfile entry are matched by: the
+ * PEP 503 normalised name and the version.
+ *
+ * @param {Object} p Component with `name` and `version`
+ * @returns {string|undefined} The key, or undefined without a name or version
+ */
+export function installedMetadataKey(p) {
+  const name = normalizePep503Name(p?.name);
+  const version = typeof p?.version === "string" ? p.version.trim() : "";
+  return name && version ? `${name}@${version}` : undefined;
+}
+
+/**
+ * Enrich lockfile entries from the installed distributions on this machine,
+ * before PyPI is asked.
+ *
+ * Each entry of `installedMetadata` is a component as
+ * {@link parseBdistMetadata} builds it from an installed `*.dist-info/METADATA`
+ * file. An entry is matched to the lockfile component with the same PEP 503
+ * normalised name and the same version, and fills the fields it lacks, so the
+ * registry is left only what the installed distributions do not answer. An
+ * installed release of another version describes a different release, so it
+ * enriches nothing. The distributions enrich the entries in place; no new
+ * components are created.
+ *
+ * @param {Object[]} pkgList Lockfile components, enriched in place
+ * @param {Object[]} installedMetadata Components parsed from installed
+ *   `*.dist-info/METADATA` files
+ * @returns {Set<string>} The keys of the installations that enriched an entry,
+ *   as {@link installedMetadataKey} builds them, so a caller that also walks
+ *   the METADATA files can skip those as components
+ */
+export function enrichFromInstalledMetadata(pkgList, installedMetadata) {
+  const byKey = new Map();
+  for (const p of pkgList || []) {
+    const key = installedMetadataKey(p);
+    if (key && !byKey.has(key)) {
+      byKey.set(key, p);
+    }
+  }
+  const enriched = new Set();
+  for (const md of installedMetadata || []) {
+    const key = installedMetadataKey(md);
+    const p = key ? byKey.get(key) : undefined;
+    if (!p) {
+      continue;
+    }
+    enriched.add(key);
+    if (!p.description?.trim() && md.description?.trim()) {
+      p.description = md.description;
+    }
+    if (!p.license && md.licenses?.length) {
+      // The licence field holds the names a lockfile consumer expects; the
+      // CycloneDX licence objects travel in `licenses`.
+      p.license = md.licenses
+        .map((alicense) => alicense.expression || alicense.license?.name)
+        .filter((aname) => aname?.length);
+      p.licenses = md.licenses;
+    }
+    if (!p.homepage && md.homepage) {
+      p.homepage = md.homepage;
+    }
+    if (!p.repository && md.repository) {
+      p.repository = md.repository;
+    }
+    if (!p.author && md.author) {
+      p.author = md.author;
+    }
+  }
+  return enriched;
+}
+
 /**
  * Method to parse python lock files such as poetry.lock, pdm.lock, uv.lock, and pylock.toml.
  *
  * @param {string} lockData Raw TOML text from poetry.lock, pdm.lock, uv.lock, or pylock.toml
  * @param {string} lockFile Lock file name for evidence
  * @param {string} pyProjectFile pyproject.toml file
+ * @param {Object} [options] Options
+ * @param {Object[]} [options.installedMetadata] Components parsed from the
+ *   installed `*.dist-info/METADATA` files of this machine, which answer
+ *   before PyPI is asked
  */
-export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
+export async function parsePyLockData(
+  lockData,
+  lockFile,
+  pyProjectFile,
+  options = {},
+) {
   let pkgList = [];
   const rootList = [];
   const dependenciesList = [];
@@ -940,7 +1052,7 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
   }
   let lockTomlObj;
   try {
-    lockTomlObj = toml.parse(lockData);
+    lockTomlObj = parseToml(lockData);
   } catch (err) {
     if (lockFile) {
       // The failing file names the manager whose lock command regenerates
@@ -1416,6 +1528,9 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
       dependsOn: [...dependsOnList].sort(),
     });
   }
+  // The installed distributions answer first, so PyPI is asked only about the
+  // packages they do not cover.
+  enrichFromInstalledMetadata(pkgList, options.installedMetadata);
   pkgList = await getPyMetadata(pkgList, false);
   return {
     parentComponent,
@@ -1427,16 +1542,33 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
   };
 }
 
+/**
+ * First version clause of a PEP 508 requirement: an operator and the version
+ * that follows it.
+ *
+ * `===` (arbitrary equality) is tried before `==`, whitespace may separate the
+ * operator from the version, and a version may carry a PEP 440 epoch (`1!2.0`)
+ * or a local version label (`2.1.0+cpu`).
+ */
+const PEP440_VERSION_CLAUSE = /(===|==|!=|<=|>=|~=|<|>)\s*([0-9A-Za-z.*_+!-]*)/;
+
 /**
  * Method to parse requirements.txt file. This must be replaced with atom parsedeps.
  *
  * @param {String} reqFile Requirements.txt file
  * @param {Boolean} fetchDepsInfo Fetch dependencies info from pypi
+ * @param {Object[]} [installedMetadata] Components parsed from the installed
+ *   `*.dist-info/METADATA` files of this machine, which answer before PyPI
+ *   is asked
  *
  * @returns {Promise[Array<Object>]} List of direct dependencies from the requirements file
  */
-export async function parseReqFile(reqFile, fetchDepsInfo = false) {
-  return await parseReqData(reqFile, null, fetchDepsInfo);
+export async function parseReqFile(
+  reqFile,
+  fetchDepsInfo = false,
+  installedMetadata = undefined,
+) {
+  return await parseReqData(reqFile, null, fetchDepsInfo, installedMetadata);
 }
 
 const LICENSE_ID_COMMENTS_PATTERN =
@@ -1465,9 +1597,17 @@ function parseLicenseComment(comment) {
  * @param {String} reqFile Requirements.txt file
  * @param {Object} reqData Requirements.txt data for internal invocations from setup.py file etc.
  * @param {Boolean} fetchDepsInfo Fetch dependencies info from pypi
+ * @param {Object[]} [installedMetadata] Components parsed from the installed
+ *   `*.dist-info/METADATA` files of this machine, which answer before PyPI
+ *   is asked
  * @returns {Promise<Array<Object>>} List of direct dependencies from the requirements file
  */
-async function parseReqData(reqFile, reqData = null, fetchDepsInfo = false) {
+async function parseReqData(
+  reqFile,
+  reqData = null,
+  fetchDepsInfo = false,
+  installedMetadata = undefined,
+) {
   const pkgList = [];
   let compScope;
   if (!reqFile && !reqData) {
@@ -1513,10 +1653,13 @@ async function parseReqData(reqFile, reqData = null, fetchDepsInfo = false) {
       continue;
     }
     let comment = null;
-    const commentMatch = l.match(/\s+#(.*)$/);
-    if (commentMatch) {
-      comment = commentMatch[1].trim();
-      l = l.substring(0, commentMatch.index).trim();
+    // Equivalent to /\s+#(.*)$/ but linear: an unanchored `\s+` backtracks
+    // quadratically on long whitespace runs without a '#', which a crafted
+    // requirements.txt used to burn seconds of CPU per line.
+    const hashIndex = l.indexOf("#");
+    if (hashIndex > 0 && /\s/.test(l[hashIndex - 1])) {
+      comment = l.substring(hashIndex + 1).trim();
+      l = l.substring(0, hashIndex).trim();
     }
     const properties = reqFile
       ? [
@@ -1607,9 +1750,7 @@ async function parseReqData(reqFile, reqData = null, fetchDepsInfo = false) {
       if (PYTHON_STD_MODULES.includes(name)) {
         continue;
       }
-      const versionMatch = versionSpecifiers.match(
-        /(==|!=|<=|>=|<|>|~=)([0-9.a-zA-Z_*\-+]*)/,
-      );
+      const versionMatch = versionSpecifiers.match(PEP440_VERSION_CLAUSE);
       let version = null;
       if (versionMatch) {
         version = versionMatch[2].replaceAll("*", "0") || null;
@@ -1636,7 +1777,7 @@ async function parseReqData(reqFile, reqData = null, fetchDepsInfo = false) {
           value: extras.join(","),
         });
       }
-      if (versionSpecifiers && !versionSpecifiers.startsWith("==")) {
+      if (versionSpecifiers && !versionSpecifiers.trim().startsWith("==")) {
         properties.push({
           name: "cdx:pypi:versionSpecifiers",
           value: versionSpecifiers.trim(),
@@ -1673,9 +1814,7 @@ async function parseReqData(reqFile, reqData = null, fetchDepsInfo = false) {
       if (PYTHON_STD_MODULES.includes(name)) {
         continue;
       }
-      const versionMatch = versionSpecifiers.match(
-        /(==|!=|<=|>=|<|>|~=)([0-9.a-zA-Z_*\-+]*)/,
-      );
+      const versionMatch = versionSpecifiers.match(PEP440_VERSION_CLAUSE);
       let version = null;
       if (versionMatch) {
         version = versionMatch[2].replaceAll("*", "0") || null;
@@ -1691,7 +1830,7 @@ async function parseReqData(reqFile, reqData = null, fetchDepsInfo = false) {
       if (licenses) {
         apkg.licenses = licenses;
       }
-      if (versionSpecifiers && !versionSpecifiers.startsWith("==")) {
+      if (versionSpecifiers && !versionSpecifiers.trim().startsWith("==")) {
         properties.push({
           name: "cdx:pypi:versionSpecifiers",
           value: versionSpecifiers.trim(),
@@ -1721,6 +1860,9 @@ async function parseReqData(reqFile, reqData = null, fetchDepsInfo = false) {
       pkgList.push(apkg);
     }
   }
+  // The installed distributions answer first, so PyPI is asked only about
+  // the packages they do not cover.
+  enrichFromInstalledMetadata(pkgList, installedMetadata);
   return await getPyMetadata(pkgList, fetchDepsInfo);
 }
 
@@ -2031,7 +2173,7 @@ export function parsePixiTomlFile(pixiToml) {
   const pixiTomlFile = readFileSync(pixiToml, { encoding: "utf-8" });
   let tomlData;
   try {
-    tomlData = toml.parse(pixiTomlFile);
+    tomlData = parseToml(pixiTomlFile);
   } catch (err) {
     console.log(`Error while parsing the pixi file ${pixiToml}.`, err);
     return {};
@@ -2107,7 +2249,7 @@ export function generatePixiLockFile(_path) {
 export function parseMojoProject(mojoProjectFile) {
   let manifest;
   try {
-    manifest = toml.parse(readFileSync(mojoProjectFile, "utf-8"));
+    manifest = parseToml(readFileSync(mojoProjectFile, "utf-8"));
   } catch (error) {
     console.warn(`Failed to parse ${mojoProjectFile}: ${error.message}`);
     return { pkgList: [], parentComponent: {} };
@@ -2201,6 +2343,7 @@ function normaliseMojoVersion(spec) {
   const trimmed = spec.trim();
   if (!trimmed) return undefined;
   // Strip common comparison operators and take the first concrete version.
-  const match = trimmed.match(/(\d+(?:\.\d+)*(?:[+-][A-Za-z0-9.]+)?)/);
+  // A PEP 440 epoch (`1!2.0`) is part of the version, not a separator.
+  const match = trimmed.match(/((?:\d+!)?\d+(?:\.\d+)*(?:[+-][A-Za-z0-9.]+)?)/);
   return match ? match[1] : undefined;
 }

package/lib/ecosystems/parsers-rust.js

@@ -3,12 +3,12 @@ import { sep as _sep, basename, dirname, join, resolve } from "node:path";
 
 import { build } from "@cdxgen/cdx-purl";
 import { globSync } from "glob";
-import * as toml from "smol-toml";
 
 import { DEBUG_MODE } from "../core/activity.js";
 import { shouldFetchLicense } from "../core/env.js";
 import { safeExistsSync } from "../core/fs.js";
 import { traceLog } from "../core/logger.js";
+import { parseToml } from "../core/toml.js";
 import { getCratesMetadata, normalizeCargoIntegrity } from "./ecosystems.js";
 
 /**
@@ -204,7 +204,7 @@ function readCargoTomlData(cargoTomlFile) {
     return undefined;
   }
   try {
-    return toml.parse(readFileSync(cargoTomlFile, { encoding: "utf-8" }));
+    return parseToml(readFileSync(cargoTomlFile, { encoding: "utf-8" }));
   } catch (error) {
     traceLog("cargo", {
       cargoTomlFile,
@@ -1082,7 +1082,7 @@ export async function parseCargoData(
     return pkgList;
   }
 
-  const cargoData = toml.parse(
+  const cargoData = parseToml(
     readFileSync(cargoLockFile, { encoding: "utf-8" }),
   );
   if (!cargoData) {
@@ -1290,7 +1290,7 @@ export function parseCargoManifestDependencyData(cargoTomlFile, context = {}) {
  */
 export function parseCargoDependencyData(cargoLockData) {
   const purlFromPackageInfo = (pkg) => decodeURIComponent(buildCargoPurl(pkg));
-  const cargoData = toml.parse(cargoLockData);
+  const cargoData = parseToml(cargoLockData);
   const packageNode = cargoData?.package;
   if (!packageNode || !Array.isArray(packageNode)) {
     return [];
@@ -1309,15 +1309,20 @@ export function parseCargoDependencyData(cargoLockData) {
       );
     }
   });
-  // Create a map of package names to package objects
-  const pkgMap = pkgList.reduce((acc, item) => {
-    acc[item.name] = item;
-    return acc;
-  }, {});
-  const result = [];
-  // parse dependency version
-  Object.values(pkgMap).forEach((pkg) => {
-    const dependsOn = new Set();
+  // A lock file can hold several versions of one crate. Cargo then names the
+  // version in every reference to it (`getrandom 0.3.4`), and a bare name is
+  // left only for crates with a single version.
+  const pkgsByName = new Map();
+  for (const pkg of pkgList) {
+    pkgsByName.set(pkg.name, [...(pkgsByName.get(pkg.name) || []), pkg]);
+  }
+  // Every package gets its own entry, so each version keeps its edges. The
+  // same name and version from two sources share a purl; their edges merge.
+  const edgesByRef = new Map();
+  for (const pkg of pkgList) {
+    const ref = purlFromPackageInfo(pkg);
+    const dependsOn = edgesByRef.get(ref) || new Set();
+    edgesByRef.set(ref, dependsOn);
     pkg.dependencies?.forEach((dep) => {
       if (dep.indexOf(" ") !== -1) {
         // fill version in dependency definition like `libc 0.2.79`
@@ -1328,20 +1333,21 @@ export function parseCargoDependencyData(cargoLockData) {
             version: depSplit[1].trim(),
           }),
         );
-      } else if (pkgMap[dep]) {
-        dependsOn.add(purlFromPackageInfo(pkgMap[dep]));
+      } else if (pkgsByName.get(dep)?.length === 1) {
+        dependsOn.add(purlFromPackageInfo(pkgsByName.get(dep)[0]));
       } else if (DEBUG_MODE) {
         console.warn(
-          `The package "${dep.name}" appears as a dependency to "${pkg.name}" but is not itself listed in the Cargo.lock file. The Cargo.lock file is invalid! The produced SBOM will not list ${dep.name} as a dependency.`,
+          pkgsByName.has(dep)
+            ? `The package "${dep}" appears as a dependency to "${pkg.name}" without a version, but the Cargo.lock file lists more than one version of it. The produced SBOM will not list ${dep} as a dependency.`
+            : `The package "${dep}" appears as a dependency to "${pkg.name}" but is not itself listed in the Cargo.lock file. The Cargo.lock file is invalid! The produced SBOM will not list ${dep} as a dependency.`,
         );
       }
     });
-    result.push({
-      ref: purlFromPackageInfo(pkg),
-      dependsOn: [...dependsOn],
-    });
-  });
-  return result;
+  }
+  return [...edgesByRef].map(([ref, dependsOn]) => ({
+    ref,
+    dependsOn: [...dependsOn],
+  }));
 }
 
 /**