@cdxgen/cdxgen 13.0.1 → 13.2.0

package/lib/ecosystems/parsers-perl.js

@@ -0,0 +1,277 @@
+import { readFileSync } from "node:fs";
+
+import { tryBuildPurl } from "../inventory/purl.js";
+
+/**
+ * Perl parser for `cpanfile` and `cpanfile.snapshot`.
+ *
+ * Carton snapshots the resolved install set as a text document: one entry per
+ * distribution, each carrying the `pathname` it was fetched from (which
+ * encodes the PAUSE author id), the packages it provides, and the
+ * requirements it was resolved against. The declared `cpanfile` lists the
+ * project's own requirements.
+ *
+ * Distributions are identified with the registered `cpan` purl type, whose
+ * rules require the author id as the namespace
+ * (`pkg:cpan/<AUTHOR>/<Distribution>@<version>`).
+ */
+
+/**
+ * Parse a `cpanfile.snapshot` into distributions with a dependency graph.
+ *
+ * @param {string} snapshotFile Path to `cpanfile.snapshot`
+ * @param {string} [cpanfileFile] Path to `cpanfile`, if present
+ * @returns {{ pkgList: object[], dependencies: object[] }}
+ */
+export function parseCpanSnapshot(snapshotFile, cpanfileFile) {
+  let text;
+  try {
+    text = readFileSync(snapshotFile, "utf-8");
+  } catch (error) {
+    console.warn(`Failed to read ${snapshotFile}: ${error.message}`);
+    return { pkgList: [], dependencies: [] };
+  }
+
+  const distributions = [];
+  let current;
+  let section;
+  for (const rawLine of text.split("\n")) {
+    if (!rawLine.trim() || rawLine.trim().startsWith("#")) {
+      continue;
+    }
+    const indent = rawLine.length - rawLine.trimStart().length;
+    const line = rawLine.trim();
+    if (indent === 0) {
+      // Top-level headers separate the snapshot sections.
+      current = undefined;
+      section = line === "DISTRIBUTIONS" ? "distributions" : undefined;
+      continue;
+    }
+    if (line.startsWith("DISTRIBUTIONS")) {
+      section = "distributions";
+      continue;
+    }
+    if (section !== "distributions") {
+      continue;
+    }
+    // A distribution line is indented by two spaces and carries
+    // `<Name>-<version>`; its details follow indented deeper.
+    if (indent === 2) {
+      const dist = parseDistributionLine(line);
+      if (dist) {
+        distributions.push(dist);
+        current = dist;
+      }
+      continue;
+    }
+    if (!current) {
+      continue;
+    }
+    if (line.startsWith("pathname:")) {
+      current.author = authorFromPathname(
+        line.slice("pathname:".length).trim(),
+      );
+      continue;
+    }
+    if (line.startsWith("provides:")) {
+      current.inProvides = true;
+      current.inRequirements = false;
+      continue;
+    }
+    if (line.startsWith("requirements:")) {
+      current.inRequirements = true;
+      current.inProvides = false;
+      continue;
+    }
+    // Deeper indented `key value` lines under provides/requirements.
+    const pair = line.match(/^([A-Za-z0-9_:.-]+)\s+(\S+)$/u);
+    if (pair) {
+      if (current.inProvides) {
+        current.provides.set(pair[1], pair[2]);
+      } else if (current.inRequirements) {
+        current.requirements.push(pair[1]);
+      }
+    }
+  }
+
+  const directNames = cpanfileFile
+    ? directRequirementsFromCpanfile(cpanfileFile)
+    : new Set();
+
+  // Map every provided package name to its distribution so requirements can
+  // be resolved to edges between distributions.
+  const distByPackage = new Map();
+  for (const dist of distributions) {
+    for (const provided of dist.provides.keys()) {
+      if (!distByPackage.has(provided)) {
+        distByPackage.set(provided, dist);
+      }
+    }
+  }
+
+  const pkgList = [];
+  const refByDist = new Map();
+  for (const dist of distributions) {
+    const name = dist.name;
+    const version = dist.version;
+    const direct =
+      directNames.size === 0
+        ? true
+        : [...dist.provides.keys()].some((pkg) => directNames.has(pkg));
+    const pkg = cpanPackage(name, version, dist.author, {
+      direct,
+      srcFile: snapshotFile,
+    });
+    pkgList.push(pkg);
+    refByDist.set(dist, pkg["bom-ref"]);
+  }
+
+  const dependencies = [];
+  for (const dist of distributions) {
+    const ref = refByDist.get(dist);
+    const dependsOn = new Set();
+    for (const requirement of dist.requirements) {
+      const target = distByPackage.get(requirement);
+      if (target && target !== dist) {
+        const targetRef = refByDist.get(target);
+        if (targetRef) {
+          dependsOn.add(targetRef);
+        }
+      }
+    }
+    dependencies.push({ ref, dependsOn: [...dependsOn].sort() });
+  }
+  return { pkgList, dependencies };
+}
+
+/**
+ * Parse a `Name-Version` distribution line.
+ *
+ * @param {string} line Trimmed distribution line
+ * @param {string} srcFile Source path for provenance
+ * @returns {object|undefined} Distribution accumulator
+ */
+function parseDistributionLine(line) {
+  const match = line.match(/^([A-Za-z0-9_.-]+?)-(\d[^ ]*)$/u);
+  if (!match) {
+    return undefined;
+  }
+  return {
+    name: match[1],
+    version: match[2],
+    author: undefined,
+    provides: new Map(),
+    requirements: [],
+    inProvides: false,
+    inRequirements: false,
+  };
+}
+
+/**
+ * Extract the PAUSE author id from a CPAN pathname.
+ *
+ * Pathnames look like `M/MI/MIYAGAWA/Plack-1.0051.tar.gz`; the author id is
+ * the third segment.
+ *
+ * @param {string} pathname CPAN pathname
+ * @returns {string|undefined} Author id
+ */
+function authorFromPathname(pathname) {
+  const segments = pathname.split("/").filter(Boolean);
+  return segments.length >= 3 ? segments[2] : undefined;
+}
+
+/**
+ * Build a component-like package record for a CPAN distribution.
+ *
+ * @param {string} name Distribution name
+ * @param {string|undefined} version Distribution version
+ * @param {string|undefined} author PAUSE author id
+ * @param {object} opts Extra context (`direct`, `srcFile`)
+ * @returns {object} Package record
+ */
+function cpanPackage(name, version, author, opts) {
+  // The cpan purl type requires the author namespace; without it the purl is
+  // dropped rather than emitted without its identity anchor.
+  const purl = author
+    ? tryBuildPurl({
+        type: "cpan",
+        namespace: author,
+        name,
+        version: version || undefined,
+      })
+    : null;
+  const properties = [
+    { name: "internal:SrcFile", value: opts.srcFile },
+    {
+      name: "cdx:cpan:dependency",
+      value: opts.direct ? "direct" : "transitive",
+    },
+  ];
+  const pkg = {
+    name,
+    ...(version ? { version } : {}),
+    type: "library",
+    scope: "required",
+    properties,
+  };
+  if (purl) {
+    pkg.purl = purl;
+    pkg["bom-ref"] = decodeURIComponent(purl);
+  } else {
+    pkg["bom-ref"] = `library:${name}:${version || ""}`;
+  }
+  return pkg;
+}
+
+/**
+ * Collect the requirement names declared in a `cpanfile`.
+ *
+ * Only the `requires` keyword outside phase blocks is read; phases such as
+ * `on 'test'` are ignored so test-only modules do not become direct
+ * dependencies.
+ *
+ * @param {string} cpanfileFile Path to `cpanfile`
+ * @returns {Set<string>} Declared requirement names
+ */
+export function directRequirementsFromCpanfile(cpanfileFile) {
+  const names = new Set();
+  let text;
+  try {
+    text = readFileSync(cpanfileFile, "utf-8");
+  } catch (error) {
+    console.warn(`Failed to read ${cpanfileFile}: ${error.message}`);
+    return names;
+  }
+  let inPhaseBlock = false;
+  let blockDepth = 0;
+  for (const rawLine of text.split("\n")) {
+    const line = rawLine.trim();
+    if (!line || line.startsWith("#")) {
+      continue;
+    }
+    if (inPhaseBlock) {
+      for (const char of line) {
+        if (char === "{") {
+          blockDepth += 1;
+        } else if (char === "}") {
+          blockDepth -= 1;
+          if (blockDepth === 0) {
+            inPhaseBlock = false;
+          }
+        }
+      }
+      continue;
+    }
+    if (/^on\s+['"]/.test(line) && line.includes("sub")) {
+      inPhaseBlock = true;
+      blockDepth = line.includes("{") ? 1 : 0;
+      continue;
+    }
+    const requires = line.match(/^requires\s+['"]([^'"]+)['"]/u);
+    if (requires) {
+      names.add(requires[1]);
+    }
+  }
+  return names;
+}

package/lib/ecosystems/parsers-python.js

@@ -4,10 +4,11 @@ import process from "node:process";
 import { URL } from "node:url";
 
 import { build } from "@cdxgen/cdx-purl";
-import toml from "@iarna/toml";
+import * as toml from "smol-toml";
 import { parse as _load } from "yaml";
 
 import { DEBUG_MODE } from "../core/activity.js";
+import { recordDegradation } from "../core/buildLedger.js";
 import { getAllFiles, safeExistsSync, safeSpawnSync } from "../core/fs.js";
 import { PYTHON_STD_MODULES } from "../core/state.js";
 import { tryBuildPurl } from "../inventory/purl.js";
@@ -27,6 +28,41 @@ import {
   normalizePyLockRegistry,
 } from "./pylockutils.js";
 
+/**
+ * TOML lock files a specific Python manager owns, mapped to the manager and
+ * the remediation id that ranks its repair. The failing file itself is the
+ * detection: the lock command that regenerates it belongs to the manager
+ * that wrote it.
+ *
+ * @type {Record<string, {manager: string, remediationId: string}>}
+ */
+const PY_LOCKFILE_DEGRADATIONS = {
+  "uv.lock": {
+    manager: "uv",
+    remediationId: "python.lockfile-unparseable.uv",
+  },
+  "poetry.lock": {
+    manager: "poetry",
+    remediationId: "python.lockfile-unparseable.poetry",
+  },
+  "pdm.lock": {
+    manager: "pdm",
+    remediationId: "python.lockfile-unparseable.pdm",
+  },
+};
+
+/**
+ * The repair for a TOML lock file that names no single manager, such as
+ * PEP 751's `pylock.toml`. Its lock command carries the manager variable, so
+ * the shaping fills in whichever manager the project turns out to use.
+ *
+ * @type {{manager: string, remediationId: string}}
+ */
+const GENERIC_PY_LOCKFILE_DEGRADATION = {
+  manager: "python",
+  remediationId: "python.lockfile-unparseable",
+};
+
 /**
  * Method to parse python requires_dist attribute found in pypi setup.py
  *
@@ -444,7 +480,12 @@ export function parsePyProjectTomlFile(tomlFile) {
     for (const agroup of Object.keys(tomlData["dependency-groups"])) {
       tomlData["dependency-groups"][agroup].forEach((p) => {
         if (typeof p === "string" || p instanceof String) {
-          const pname = p.split(/(==|<=|~=|>=)/)[0].split(" ")[0];
+          const pname = normalizePythonDependencyKey(
+            p.split(/(==|<=|~=|>=)/)[0].split(" ")[0],
+          );
+          if (!pname) {
+            return;
+          }
           if (!groupDepsKeys[pname]) {
             groupDepsKeys[pname] = [];
           }
@@ -466,8 +507,13 @@ export function parsePyProjectTomlFile(tomlFile) {
   }
   if (tomlData?.tool?.poetry?.dependencies) {
     for (const adep of Object.keys(tomlData?.tool?.poetry?.dependencies)) {
+      // poetry keys keep the author's casing (Click = "^8.1"), while every
+      // consumer of directDepsKeys looks entries up with the normalised
+      // (lowercase, hyphenated) package name, so normalise at the source.
+      const poetryDepKey = normalizePythonDependencyKey(adep);
       if (
-        ![
+        !poetryDepKey ||
+        [
           "python",
           "py",
           "pytest",
@@ -475,43 +521,50 @@ export function parsePyProjectTomlFile(tomlFile) {
           "ruff",
           "setuptools",
           "bandit",
-        ].includes(adep)
+        ].includes(poetryDepKey)
       ) {
-        directDepsKeys[adep] = true;
-        const poetryDependency = tomlData.tool.poetry.dependencies[adep];
-        if (poetryDependency?.git) {
-          recordPythonDependencySource(
-            dependencySourceMap,
-            adep,
-            "git",
-            poetryDependency.git,
-          );
-        } else if (poetryDependency?.url) {
-          recordPythonDependencySource(
-            dependencySourceMap,
-            adep,
-            "url",
-            poetryDependency.url,
-          );
-        } else if (poetryDependency?.path) {
-          recordPythonDependencySource(
-            dependencySourceMap,
-            adep,
-            "path",
-            poetryDependency.path,
-          );
-        }
+        continue;
+      }
+      directDepsKeys[poetryDepKey] = true;
+      const poetryDependency = tomlData.tool.poetry.dependencies[adep];
+      if (poetryDependency?.git) {
+        recordPythonDependencySource(
+          dependencySourceMap,
+          adep,
+          "git",
+          poetryDependency.git,
+        );
+      } else if (poetryDependency?.url) {
+        recordPythonDependencySource(
+          dependencySourceMap,
+          adep,
+          "url",
+          poetryDependency.url,
+        );
+      } else if (poetryDependency?.path) {
+        recordPythonDependencySource(
+          dependencySourceMap,
+          adep,
+          "path",
+          poetryDependency.path,
+        );
       }
     } // for
+    // Group keys are matched against the normalised package names from the
+    // lock file, so they are normalised here too.
     if (tomlData?.tool?.poetry?.group) {
       for (const agroup of Object.keys(tomlData.tool.poetry.group)) {
         for (const adep of Object.keys(
           tomlData.tool.poetry.group[agroup]?.dependencies,
         )) {
-          if (!groupDepsKeys[adep]) {
-            groupDepsKeys[adep] = [];
+          const poetryGroupDepKey = normalizePythonDependencyKey(adep);
+          if (!poetryGroupDepKey) {
+            continue;
           }
-          groupDepsKeys[adep].push(agroup);
+          if (!groupDepsKeys[poetryGroupDepKey]) {
+            groupDepsKeys[poetryGroupDepKey] = [];
+          }
+          groupDepsKeys[poetryGroupDepKey].push(agroup);
           const poetryDependency =
             tomlData.tool.poetry.group[agroup]?.dependencies?.[adep];
           if (poetryDependency?.git) {
@@ -780,7 +833,27 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
     dependencySourceMap = pyProjMap.dependencySourceMap || {};
     parentComponent = pyProjMap.parentComponent;
     workspacePaths = pyProjMap.workspacePaths;
-    if (workspacePaths?.length) {
+    // `parsePyProjectTomlFile` reports a parse failure and returns an empty map, so a
+    // pyproject.toml this parser cannot read leaves `parentComponent` undefined while
+    // the lock file still declares workspaces. Building the workspace tree then throws
+    // on the first dereference and takes the whole SBOM with it. Degrade to a flattened
+    // tree instead - the caller supplies its own parent component, so the components
+    // themselves are still reported.
+    if (workspacePaths?.length && !parentComponent) {
+      console.log(
+        `Unable to read a parent component from ${pyProjectFile}, so the workspace structure declared in ${basename(lockFile)} cannot be reconstructed.`,
+      );
+      console.log(
+        "The dependency tree in the generated SBOM will be flattened and therefore incorrect.",
+      );
+      recordDegradation("python.pyproject-unparseable", {
+        ecosystem: "python",
+        impact: "transitive-deps",
+        path: pyProjectFile,
+        detail: `${basename(pyProjectFile)} could not be parsed, so the workspace structure declared in ${basename(lockFile)} was not reconstructed.`,
+      });
+    }
+    if (workspacePaths?.length && parentComponent) {
       if (!hasWorkspaces) {
         hasWorkspaces = true;
       }
@@ -840,7 +913,12 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
               ] = awpyproj;
             }
           }
-          const wparentComponentRef = wcompMap.parentComponent["bom-ref"];
+          // The block above already treats an unreadable workspace pyproject.toml as
+          // tolerable, so this must not dereference the component it skipped. A
+          // workspace member whose manifest cannot be parsed contributes no parent
+          // ref, and recording `undefined` as the parent of its dependencies would
+          // corrupt the tree for the members that did parse.
+          const wparentComponentRef = wcompMap?.parentComponent?.["bom-ref"];
           if (wcompMap?.dependencySourceMap) {
             Object.assign(
               workspaceDependencySourceMap,
@@ -848,7 +926,7 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
             );
           }
           // Track the parents of workspace direct dependencies
-          if (wcompMap?.directDepsKeys) {
+          if (wcompMap?.directDepsKeys && wparentComponentRef) {
             for (const wdd of Object.keys(wcompMap?.directDepsKeys)) {
               if (!pkgParentMap[wdd]) {
                 pkgParentMap[wdd] = [];
@@ -865,6 +943,20 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
     lockTomlObj = toml.parse(lockData);
   } catch (err) {
     if (lockFile) {
+      // The failing file names the manager whose lock command regenerates
+      // it. A standards-named lock file (PEP 751 `pylock.toml`) belongs to
+      // no single manager, so it falls back to the generic repair, whose
+      // lock command the shaping resolves from whichever manager the
+      // project turns out to use.
+      const degradation =
+        PY_LOCKFILE_DEGRADATIONS[basename(lockFile)] ||
+        GENERIC_PY_LOCKFILE_DEGRADATION;
+      recordDegradation(degradation.remediationId, {
+        ecosystem: "python",
+        impact: "transitive-deps",
+        path: lockFile,
+        detail: `The ${degradation.manager} lock file could not be parsed, so no locked versions were captured from it.`,
+      });
       console.log(`Error while parsing the lock file ${lockFile}.`, err);
     } else {
       console.log("Error while parsing the lock data as toml", err);
@@ -1218,6 +1310,38 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
       }
     }
   }
+  // Seed the parent component's first-level edges from the manifest. Lock
+  // files describe the resolved closure but do not say which of the locked
+  // packages the root project directly depends on; parsePyProjectTomlFile
+  // derived that set into directDepsKeys from [project.dependencies] /
+  // [tool.poetry.dependencies]. Only packages that are locked resolve to a
+  // bom-ref here, so the seeded edge never dangles and dev-group or
+  // unresolved dependencies stay out of the first level. Seeding
+  // depsMap (instead of pushing into dependenciesList) lets the
+  // materialisation below apply its usual ref resolution and sorting.
+  if (parentComponent?.["bom-ref"] && Object.keys(directDepsKeys).length) {
+    const rootRef = parentComponent["bom-ref"];
+    const rootDeps = new Set();
+    for (const depKey of Object.keys(directDepsKeys)) {
+      const depRef =
+        existingPkgMap[depKey] ||
+        existingPkgMap[depKey.replaceAll("-", "_")] ||
+        existingPkgMap[`py${depKey}`];
+      if (depRef && depRef !== rootRef) {
+        rootDeps.add(depRef);
+      }
+    }
+    if (rootDeps.size) {
+      const existingRootDeps = depsMap[rootRef];
+      if (existingRootDeps) {
+        for (const adep of rootDeps) {
+          existingRootDeps.add(adep);
+        }
+      } else {
+        depsMap[rootRef] = rootDeps;
+      }
+    }
+  }
   for (const key of Object.keys(depsMap)) {
     const dependsOnList = new Set();
     const parentPkg = pkgBomRefMap[key];
@@ -1921,8 +2045,20 @@ export function parsePixiTomlFile(pixiToml) {
   pkg.name = projectData.name;
   pkg.version = projectData.version;
   // pkg.authors = tomlData['project']['authors'];
-  pkg.homepage = projectData.homepage;
-  pkg.repository = projectData.repository;
+  // The parent component only gets `cleanParentComponent`, which converts the
+  // transient `{ url }` shape — raw strings would be dropped silently, so
+  // emit external references directly instead.
+  if (projectData.homepage) {
+    pkg.externalReferences = pkg.externalReferences || [];
+    pkg.externalReferences.push({
+      type: "website",
+      url: projectData.homepage,
+    });
+  }
+  if (projectData.repository) {
+    pkg.externalReferences = pkg.externalReferences || [];
+    pkg.externalReferences.push({ type: "vcs", url: projectData.repository });
+  }
   return pkg;
 }