@cdxgen/cdxgen 13.3.0 → 13.5.0

package/lib/ecosystems/parsers-python.js

@@ -1,16 +1,15 @@
 import { readFileSync } from "node:fs";
 import { basename, dirname, join, relative } from "node:path";
-import process from "node:process";
 import { URL } from "node:url";
 
 import { build } from "@cdxgen/cdx-purl";
-import * as toml from "smol-toml";
 import { parse as _load } from "yaml";
 
 import { DEBUG_MODE } from "../core/activity.js";
 import { recordDegradation } from "../core/buildLedger.js";
 import { getAllFiles, safeExistsSync, safeSpawnSync } from "../core/fs.js";
 import { PYTHON_STD_MODULES } from "../core/state.js";
+import { parseToml } from "../core/toml.js";
 import { tryBuildPurl } from "../inventory/purl.js";
 import {
   createExternalReferenceKey,
@@ -95,8 +94,11 @@ export function parsePyRequiresDist(dist_string) {
  * Method to parse pipfile.lock data
  *
  * @param {Object} lockData JSON data from Pipfile.lock
+ * @param {Object[]} [installedMetadata] Components parsed from the installed
+ *   `*.dist-info/METADATA` files of this machine, which answer before PyPI
+ *   is asked
  */
-export async function parsePiplockData(lockData) {
+export async function parsePiplockData(lockData, installedMetadata) {
   const pkgList = [];
   Object.keys(lockData)
     .filter((i) => i !== "_meta")
@@ -112,6 +114,9 @@ export async function parsePiplockData(lockData) {
         }
       });
     });
+  // The installed distributions answer first, so PyPI is asked only about
+  // the packages they do not cover.
+  enrichFromInstalledMetadata(pkgList, installedMetadata);
   return await getPyMetadata(pkgList, false);
 }
 
@@ -274,8 +279,12 @@ function collectPythonManifestSource(pkg) {
     { kind: "url", value: pkg?.vcs?.url },
     { kind: "url", value: pkg?.source?.url },
     { kind: "path", value: pkg?.source?.path },
+    { kind: "path", value: pkg?.source?.directory },
     { kind: "path", value: pkg?.source?.editable },
     { kind: "path", value: pkg?.source?.virtual },
+    { kind: "path", value: pkg?.directory?.path },
+    { kind: "url", value: pkg?.archive?.url },
+    { kind: "path", value: pkg?.archive?.path },
   ];
   for (const candidate of sourceCandidates) {
     if (typeof candidate.value !== "string" || !candidate.value.trim()) {
@@ -395,7 +404,7 @@ export function parsePyProjectTomlFile(tomlFile) {
   const groupDepsKeys = {};
   const dependencySourceMap = {};
   try {
-    tomlData = toml.parse(readFileSync(tomlFile, { encoding: "utf-8" }));
+    tomlData = parseToml(readFileSync(tomlFile, { encoding: "utf-8" }));
   } catch (err) {
     console.log(`Error while parsing the pyproject file ${tomlFile}.`, err);
   }
@@ -788,14 +797,114 @@ function collectPythonLockMetadataDistributionReferences(fileEntries) {
   return distributionReferences;
 }
 
+/**
+ * Normalise a Python distribution name the way PEP 503 does, so an installed
+ * distribution and the lockfile entry it belongs to compare equal whatever
+ * separator and case each of them used.
+ *
+ * @param {string} value Distribution name
+ * @returns {string|undefined} The normalised name, or undefined for no name
+ */
+export function normalizePep503Name(value) {
+  if (typeof value !== "string" || !value.trim()) {
+    return undefined;
+  }
+  return value
+    .trim()
+    .toLowerCase()
+    .replaceAll(/[-_.]+/g, "-");
+}
+
+/**
+ * The key an installed distribution and a lockfile entry are matched by: the
+ * PEP 503 normalised name and the version.
+ *
+ * @param {Object} p Component with `name` and `version`
+ * @returns {string|undefined} The key, or undefined without a name or version
+ */
+export function installedMetadataKey(p) {
+  const name = normalizePep503Name(p?.name);
+  const version = typeof p?.version === "string" ? p.version.trim() : "";
+  return name && version ? `${name}@${version}` : undefined;
+}
+
+/**
+ * Enrich lockfile entries from the installed distributions on this machine,
+ * before PyPI is asked.
+ *
+ * Each entry of `installedMetadata` is a component as
+ * {@link parseBdistMetadata} builds it from an installed `*.dist-info/METADATA`
+ * file. An entry is matched to the lockfile component with the same PEP 503
+ * normalised name and the same version, and fills the fields it lacks, so the
+ * registry is left only what the installed distributions do not answer. An
+ * installed release of another version describes a different release, so it
+ * enriches nothing. The distributions enrich the entries in place; no new
+ * components are created.
+ *
+ * @param {Object[]} pkgList Lockfile components, enriched in place
+ * @param {Object[]} installedMetadata Components parsed from installed
+ *   `*.dist-info/METADATA` files
+ * @returns {Set<string>} The keys of the installations that enriched an entry,
+ *   as {@link installedMetadataKey} builds them, so a caller that also walks
+ *   the METADATA files can skip those as components
+ */
+export function enrichFromInstalledMetadata(pkgList, installedMetadata) {
+  const byKey = new Map();
+  for (const p of pkgList || []) {
+    const key = installedMetadataKey(p);
+    if (key && !byKey.has(key)) {
+      byKey.set(key, p);
+    }
+  }
+  const enriched = new Set();
+  for (const md of installedMetadata || []) {
+    const key = installedMetadataKey(md);
+    const p = key ? byKey.get(key) : undefined;
+    if (!p) {
+      continue;
+    }
+    enriched.add(key);
+    if (!p.description?.trim() && md.description?.trim()) {
+      p.description = md.description;
+    }
+    if (!p.license && md.licenses?.length) {
+      // The licence field holds the names a lockfile consumer expects; the
+      // CycloneDX licence objects travel in `licenses`.
+      p.license = md.licenses
+        .map((alicense) => alicense.expression || alicense.license?.name)
+        .filter((aname) => aname?.length);
+      p.licenses = md.licenses;
+    }
+    if (!p.homepage && md.homepage) {
+      p.homepage = md.homepage;
+    }
+    if (!p.repository && md.repository) {
+      p.repository = md.repository;
+    }
+    if (!p.author && md.author) {
+      p.author = md.author;
+    }
+  }
+  return enriched;
+}
+
 /**
  * Method to parse python lock files such as poetry.lock, pdm.lock, uv.lock, and pylock.toml.
  *
  * @param {string} lockData Raw TOML text from poetry.lock, pdm.lock, uv.lock, or pylock.toml
  * @param {string} lockFile Lock file name for evidence
  * @param {string} pyProjectFile pyproject.toml file
+ * @param {Object} [options] Options
+ * @param {Object[]} [options.installedMetadata] Components parsed from the
+ *   installed `*.dist-info/METADATA` files of this machine, which answer
+ *   before PyPI is asked
  */
-export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
+export async function parsePyLockData(
+  lockData,
+  lockFile,
+  pyProjectFile,
+  options = {},
+) {
   let pkgList = [];
   const rootList = [];
   const dependenciesList = [];
@@ -942,7 +1051,7 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
   }
   let lockTomlObj;
   try {
-    lockTomlObj = toml.parse(lockData);
+    lockTomlObj = parseToml(lockData);
   } catch (err) {
     if (lockFile) {
       // The failing file names the manager whose lock command regenerates
@@ -1418,6 +1527,9 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
       dependsOn: [...dependsOnList].sort(),
     });
   }
+  // The installed distributions answer first, so PyPI is asked only about the
+  // packages they do not cover.
+  enrichFromInstalledMetadata(pkgList, options.installedMetadata);
   pkgList = await getPyMetadata(pkgList, false);
   return {
     parentComponent,
@@ -1444,11 +1556,18 @@ const PEP440_VERSION_CLAUSE = /(===|==|!=|<=|>=|~=|<|>)\s*([0-9A-Za-z.*_+!-]*)/;
  *
  * @param {String} reqFile Requirements.txt file
  * @param {Boolean} fetchDepsInfo Fetch dependencies info from pypi
+ * @param {Object[]} [installedMetadata] Components parsed from the installed
+ *   `*.dist-info/METADATA` files of this machine, which answer before PyPI
+ *   is asked
  *
  * @returns {Promise[Array<Object>]} List of direct dependencies from the requirements file
  */
-export async function parseReqFile(reqFile, fetchDepsInfo = false) {
-  return await parseReqData(reqFile, null, fetchDepsInfo);
+export async function parseReqFile(
+  reqFile,
+  fetchDepsInfo = false,
+  installedMetadata = undefined,
+) {
+  return await parseReqData(reqFile, null, fetchDepsInfo, installedMetadata);
 }
 
 const LICENSE_ID_COMMENTS_PATTERN =
@@ -1477,9 +1596,17 @@ function parseLicenseComment(comment) {
  * @param {String} reqFile Requirements.txt file
  * @param {Object} reqData Requirements.txt data for internal invocations from setup.py file etc.
  * @param {Boolean} fetchDepsInfo Fetch dependencies info from pypi
+ * @param {Object[]} [installedMetadata] Components parsed from the installed
+ *   `*.dist-info/METADATA` files of this machine, which answer before PyPI
+ *   is asked
  * @returns {Promise<Array<Object>>} List of direct dependencies from the requirements file
  */
-async function parseReqData(reqFile, reqData = null, fetchDepsInfo = false) {
+async function parseReqData(
+  reqFile,
+  reqData = null,
+  fetchDepsInfo = false,
+  installedMetadata = undefined,
+) {
   const pkgList = [];
   let compScope;
   if (!reqFile && !reqData) {
@@ -1732,6 +1859,9 @@ async function parseReqData(reqFile, reqData = null, fetchDepsInfo = false) {
       pkgList.push(apkg);
     }
   }
+  // The installed distributions answer first, so PyPI is asked only about
+  // the packages they do not cover.
+  enrichFromInstalledMetadata(pkgList, installedMetadata);
   return await getPyMetadata(pkgList, fetchDepsInfo);
 }
 
@@ -2042,7 +2172,7 @@ export function parsePixiTomlFile(pixiToml) {
   const pixiTomlFile = readFileSync(pixiToml, { encoding: "utf-8" });
   let tomlData;
   try {
-    tomlData = toml.parse(pixiTomlFile);
+    tomlData = parseToml(pixiTomlFile);
   } catch (err) {
     console.log(`Error while parsing the pixi file ${pixiToml}.`, err);
     return {};
@@ -2074,28 +2204,30 @@ export function parsePixiTomlFile(pixiToml) {
 }
 
 /**
- * Method to run cli command `pixi install`
- *
+ * Method to run cli command `pixi install` in the project directory.
  *
+ * @param {string} projectPath Directory containing pixi.toml
+ * @returns {boolean} true when pixi install succeeded
  */
-export function generatePixiLockFile(_path) {
-  const result = safeSpawnSync("pixi", ["install"]);
+export function generatePixiLockFile(projectPath) {
+  const result = safeSpawnSync("pixi", ["install"], { cwd: projectPath });
 
   if (result.status !== 0) {
-    // Handle errors
+    // Handle errors. The caller carries on without a lock file: exiting here
+    // would also stop a long-running cdxgen server.
     if (result.error && result.error.code === "ENOENT") {
       console.error(
         "Error: pixi command not found. Make sure pixi.js is installed globally.",
       );
     } else {
       console.error(
-        `Error executing pixi install: ${result.error || result.stderr.toString()}`,
+        `Error executing pixi install: ${result.error || result.stderr?.toString()}`,
       );
     }
-    process.exit(1);
-  } else {
-    console.log("Dependencies installed successfully.");
+    return false;
   }
+  console.log("Dependencies installed successfully.");
+  return true;
 }
 
 /**
@@ -2118,7 +2250,7 @@ export function generatePixiLockFile(_path) {
 export function parseMojoProject(mojoProjectFile) {
   let manifest;
   try {
-    manifest = toml.parse(readFileSync(mojoProjectFile, "utf-8"));
+    manifest = parseToml(readFileSync(mojoProjectFile, "utf-8"));
   } catch (error) {
     console.warn(`Failed to parse ${mojoProjectFile}: ${error.message}`);
     return { pkgList: [], parentComponent: {} };

package/lib/ecosystems/parsers-rust.js

@@ -3,12 +3,12 @@ import { sep as _sep, basename, dirname, join, resolve } from "node:path";
 
 import { build } from "@cdxgen/cdx-purl";
 import { globSync } from "glob";
-import * as toml from "smol-toml";
 
 import { DEBUG_MODE } from "../core/activity.js";
 import { shouldFetchLicense } from "../core/env.js";
 import { safeExistsSync } from "../core/fs.js";
 import { traceLog } from "../core/logger.js";
+import { parseToml } from "../core/toml.js";
 import { getCratesMetadata, normalizeCargoIntegrity } from "./ecosystems.js";
 
 /**
@@ -204,7 +204,7 @@ function readCargoTomlData(cargoTomlFile) {
     return undefined;
   }
   try {
-    return toml.parse(readFileSync(cargoTomlFile, { encoding: "utf-8" }));
+    return parseToml(readFileSync(cargoTomlFile, { encoding: "utf-8" }));
   } catch (error) {
     traceLog("cargo", {
       cargoTomlFile,
@@ -1082,7 +1082,7 @@ export async function parseCargoData(
     return pkgList;
   }
 
-  const cargoData = toml.parse(
+  const cargoData = parseToml(
     readFileSync(cargoLockFile, { encoding: "utf-8" }),
   );
   if (!cargoData) {
@@ -1290,7 +1290,7 @@ export function parseCargoManifestDependencyData(cargoTomlFile, context = {}) {
  */
 export function parseCargoDependencyData(cargoLockData) {
   const purlFromPackageInfo = (pkg) => decodeURIComponent(buildCargoPurl(pkg));
-  const cargoData = toml.parse(cargoLockData);
+  const cargoData = parseToml(cargoLockData);
   const packageNode = cargoData?.package;
   if (!packageNode || !Array.isArray(packageNode)) {
     return [];