@cdxgen/cdxgen 13.5.0 → 13.7.0

package/lib/ecosystems/parsers-misc.js

@@ -1,5 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { readFileSync } from "node:fs";
+import { readdirSync, readFileSync } from "node:fs";
 import {
   sep as _sep,
   basename,
@@ -18,6 +18,7 @@ import {
   hasDangerousUnicode,
   readEnvironmentVariable,
 } from "../core/activity.js";
+import { readRegularTextFile } from "../core/boundedFile.js";
 import { recordDegradation } from "../core/buildLedger.js";
 import { deferFailOnError } from "../core/deferredExit.js";
 import { shouldFetchLicense } from "../core/env.js";
@@ -52,7 +53,12 @@ import {
   tryBuildPurl,
 } from "../inventory/purl.js";
 import { getLicenses, spdxLicenses } from "../inventory/spdx.js";
-import { cmakeFetchDependencies } from "../parsers/cmakeLists.js";
+import {
+  cmakeFetchDependencies,
+  cmakeSetVariables,
+  expandCmakeVariables,
+  parseCmakeCommands,
+} from "../parsers/cmakeLists.js";
 import { isValidIdnEmail } from "../parsers/email.js";
 import { submodulePurlCoordinates } from "../parsers/gitmodules.js";
 import { getDartMetadata } from "./ecosystems.js";
@@ -2784,6 +2790,130 @@ export function cmakeFetchDependencyComponent(dep, srcFile) {
   return component;
 }
 
+const MAX_MESON_WRAPS = 256;
+const MAX_MESON_WRAP_BYTES = 64 * 1024;
+
+function upstreamVersionOf(text) {
+  if (typeof text !== "string") {
+    return undefined;
+  }
+  const value = text.trim().replace(/^v(?=\d)/, "");
+  return /^\d+(\.\d+)*([-+._~][0-9A-Za-z.]+)?$/.test(value) ? value : undefined;
+}
+
+// `brotli-1.1.0`, `brotli-1.1.0.tar.gz`: the version after the last `-` that
+// starts a number.
+function versionFromArchiveName(name) {
+  if (typeof name !== "string") {
+    return undefined;
+  }
+  const stem = name.replace(/\.(tar\.gz|tar\.xz|tar\.bz2|tgz|zip|tar)$/i, "");
+  const dash = stem.search(/-\d[^-]*$/);
+  return dash === -1 ? undefined : upstreamVersionOf(stem.slice(dash + 1));
+}
+
+/**
+ * Read the wrap files of a meson project's subprojects directory: which
+ * dependency names each wrap provides, and the version it pins.
+ * https://mesonbuild.com/Wrap-dependency-system-manual.html
+ *
+ * A WrapDB wrap records `wrapdb_version = <upstream>-<wrap revision>`; any wrap
+ * names its unpacked `directory` and its archive, which carry the upstream
+ * version; a git wrap pins a `revision` that is often a version tag.
+ *
+ * @param {string} subprojectsDir path of the subprojects directory
+ * @returns {Map<string, {wrap: string, version: string, source: string, revision?: string}>}
+ *   by provided dependency name, lower-case
+ */
+export function readMesonWraps(subprojectsDir) {
+  const provided = new Map();
+  let wrapFiles = [];
+  try {
+    wrapFiles = readdirSync(subprojectsDir)
+      .filter((name) => name.endsWith(".wrap"))
+      .sort()
+      .slice(0, MAX_MESON_WRAPS);
+  } catch {
+    return provided;
+  }
+  for (const wrapFile of wrapFiles) {
+    // A regular file only: a FIFO named *.wrap would block a plain read.
+    const text = readRegularTextFile(
+      join(subprojectsDir, wrapFile),
+      MAX_MESON_WRAP_BYTES,
+    );
+    if (text === undefined) {
+      continue;
+    }
+    const sections = {};
+    let section = "";
+    for (const rawLine of text.split(/\r?\n/)) {
+      const line = rawLine.trim();
+      if (!line || line.startsWith("#") || line.startsWith(";")) {
+        continue;
+      }
+      if (line.startsWith("[") && line.endsWith("]")) {
+        section = line.slice(1, -1).trim();
+        sections[section] = sections[section] || {};
+        continue;
+      }
+      const equals = line.indexOf("=");
+      if (equals <= 0 || !section) {
+        continue;
+      }
+      sections[section][line.slice(0, equals).trim()] = line
+        .slice(equals + 1)
+        .trim();
+    }
+    const body = sections["wrap-file"] || sections["wrap-git"] || {};
+    let version;
+    let source;
+    let revision;
+    if (body.wrapdb_version) {
+      const dash = body.wrapdb_version.lastIndexOf("-");
+      version = upstreamVersionOf(
+        dash > 0 ? body.wrapdb_version.slice(0, dash) : body.wrapdb_version,
+      );
+      revision = dash > 0 ? body.wrapdb_version.slice(dash + 1) : undefined;
+      source = "wrapdb_version";
+    }
+    if (!version && sections["wrap-file"]) {
+      version =
+        versionFromArchiveName(body.directory) ||
+        versionFromArchiveName(body.source_filename);
+      source = version ? "directory" : undefined;
+    }
+    if (!version && sections["wrap-git"]) {
+      version = upstreamVersionOf(body.revision);
+      source = version ? "revision" : undefined;
+    }
+    if (!version) {
+      continue;
+    }
+    const wrap = wrapFile.slice(0, -".wrap".length);
+    const entry = { wrap, version, source, ...(revision ? { revision } : {}) };
+    const names = new Set([wrap.toLowerCase()]);
+    const provide = sections.provide || {};
+    for (const [key, value] of Object.entries(provide)) {
+      if (key === "dependency_names" || key === "program_names") {
+        for (const name of value.split(",")) {
+          if (name.trim()) {
+            names.add(name.trim().toLowerCase());
+          }
+        }
+      } else {
+        names.add(key.toLowerCase());
+      }
+    }
+    for (const name of names) {
+      if (!provided.has(name)) {
+        provided.set(name, entry);
+      }
+    }
+  }
+  return provided;
+}
+
 /**
  * Parse a CMake-like build file (CMakeLists.txt, meson.build, etc.) and extract the
  * parent component and list of dependency packages.
@@ -2801,6 +2931,11 @@ export function cmakeFetchDependencyComponent(dep, srcFile) {
  */
 export function parseCmakeLikeFile(cmakeListFile, pkgType, options = {}) {
   const rawData = readFileSync(cmakeListFile, { encoding: "utf-8" });
+  // The wraps a meson project ships pin the versions of the subprojects its
+  // dependency() calls fall back to.
+  const mesonWraps = basename(cmakeListFile).startsWith("meson")
+    ? readMesonWraps(join(dirname(cmakeListFile), "subprojects"))
+    : new Map();
   let cmakeListData = rawData;
   const pkgList = [];
   const pkgAddedMap = {};
@@ -2808,6 +2943,25 @@ export function parseCmakeLikeFile(cmakeListFile, pkgType, options = {}) {
   const versionsMap = {};
   let parentComponent = {};
   const templateValues = {};
+  // The parent component is built the same way whether its name and version
+  // came from the parsed project() command or from the line loop below.
+  const toParentComponent = (parentName, parentVersion) => {
+    const component = {
+      group: options.projectGroup || "",
+      name: parentName,
+      version: parentVersion || options.projectVersion || "",
+      type: "application",
+    };
+    component["purl"] = build({
+      type: pkgType,
+      namespace: component.group || null,
+      name: component.name,
+      version: component.version || null,
+      subpath: null,
+    });
+    component["bom-ref"] = decodeURIComponent(component["purl"]);
+    return component;
+  };
   // A dependency fetched at configure time is the same package a find_package()
   // of the same name asks for: the fetched component, with its source and pin,
   // stands for both.
@@ -2823,6 +2977,38 @@ export function parseCmakeLikeFile(cmakeListFile, pkgType, options = {}) {
       fetchedNames.add(dep.name.toLowerCase());
       pkgList.push(component);
     }
+    // A project() call may span several lines, which the line loop below
+    // cannot rejoin, so the parent name and version are read from the parsed
+    // commands instead. The first argument is the project name and the
+    // VERSION keyword is followed by the version.
+    const commands = parseCmakeCommands(rawData);
+    const variables = cmakeSetVariables(commands);
+    const projectCommand = commands.find(
+      (command) => command.name === "project",
+    );
+    if (projectCommand?.args.length) {
+      const args = projectCommand.args.map((arg) =>
+        expandCmakeVariables(arg, variables),
+      );
+      const parentName = args[0].replace(":", "").trim();
+      let parentVersion;
+      const versionIndex = args.findIndex(
+        (arg) => arg.toLowerCase() === "version",
+      );
+      if (versionIndex > -1 && args.length > versionIndex + 1) {
+        parentVersion = args[versionIndex + 1];
+      }
+      // A `${VAR}` that no `set()` in this file defines stays literal. It is
+      // not a version, and percent-encoding it into a purl publishes the
+      // variable name as the version, so the field is left empty instead.
+      // CMakeCache.txt carries the evaluated value when a build tree exists.
+      if (parentVersion?.includes("$")) {
+        parentVersion = undefined;
+      }
+      if (parentName.length && !parentName.includes("$")) {
+        parentComponent = toParentComponent(parentName, parentVersion);
+      }
+    }
   }
   cmakeListData = cmakeListData
     .replace(/^ {2}/g, "")
@@ -2883,23 +3069,7 @@ export function parseCmakeLikeFile(cmakeListFile, pkgType, options = {}) {
         parentVersion = undefined;
       }
       if (parentName?.length && !parentName.includes("$")) {
-        parentComponent = {
-          group: options.projectGroup || "",
-          name: parentName,
-          version: parentVersion || options.projectVersion || "",
-          type: "application",
-        };
-        const safePath = path ? path.replace(/^\/+/, "") : null;
-        parentComponent["purl"] = build({
-          type: pkgType,
-          namespace: parentComponent.group || null,
-          name: parentComponent.name,
-          version: parentComponent.version || null,
-          subpath: safePath,
-        });
-        parentComponent["bom-ref"] = decodeURIComponent(
-          parentComponent["purl"],
-        );
+        parentComponent = toParentComponent(parentName, parentVersion);
       }
     } else if (CMAKE_FIND_COMMANDS.has(command)) {
       let tmpB = commandArgs.split(")")[0].split(" ");
@@ -3041,7 +3211,26 @@ export function parseCmakeLikeFile(cmakeListFile, pkgType, options = {}) {
         // wrap match has since replaced, so the scraped name is the fallback.
         const versionSpecifier =
           versionSpecifiersMap[n] || versionSpecifiersMap[scrapedName];
-        const version = versionsMap[n] || versionsMap[scrapedName] || "";
+        let version = versionsMap[n] || versionsMap[scrapedName] || "";
+        const localWrap =
+          mesonWraps.get(scrapedName.toLowerCase()) ||
+          mesonWraps.get(n.toLowerCase());
+        if (localWrap && !version) {
+          version = localWrap.version;
+          props.push({ name: "cdx:meson:wrap", value: localWrap.wrap });
+          props.push({
+            name: "cdx:meson:wrap:versionSource",
+            value: localWrap.source,
+          });
+          if (localWrap.revision) {
+            props.push({
+              name: "cdx:meson:wrap:revision",
+              value: localWrap.revision,
+            });
+          }
+          // A pinned wrap is a stronger identity than a name match alone.
+          confidence = Math.max(confidence, 0.7);
+        }
         if (versionSpecifier) {
           props.push({
             name: "cdx:build:versionSpecifiers",

package/lib/ecosystems/parsers-python.js

@@ -8,6 +8,7 @@ import { parse as _load } from "yaml";
 import { DEBUG_MODE } from "../core/activity.js";
 import { recordDegradation } from "../core/buildLedger.js";
 import { getAllFiles, safeExistsSync, safeSpawnSync } from "../core/fs.js";
+import { sanitizeBomUrl } from "../core/propertySanitizer.js";
 import { PYTHON_STD_MODULES } from "../core/state.js";
 import { parseToml } from "../core/toml.js";
 import { tryBuildPurl } from "../inventory/purl.js";
@@ -888,6 +889,61 @@ export function enrichFromInstalledMetadata(pkgList, installedMetadata) {
   return enriched;
 }
 
+// The interpreter cdxgen assumes for a lock file's environment. A marker that
+// tests the implementation against it is decidable without a target machine.
+const IMPLEMENTATION_CLAUSE_RE =
+  /^(platform_python_implementation|implementation_name)\s*(==|!=)\s*(['"])([A-Za-z]+)\3$/;
+
+function implementationClauseIsFalse(clause) {
+  const match = IMPLEMENTATION_CLAUSE_RE.exec(clause.trim());
+  if (!match) {
+    return false;
+  }
+  const expected = match[1] === "implementation_name" ? "cpython" : "CPython";
+  const equal = match[4] === expected;
+  return match[2] === "==" ? !equal : equal;
+}
+
+/**
+ * Whether an environment marker is false for every CPython interpreter. Only
+ * `platform_python_implementation` and `implementation_name` clauses are
+ * decided; anything else, or a marker with parentheses, is treated as possibly
+ * true, so the answer errs towards keeping a package.
+ *
+ * @param {string} marker PEP 508 environment marker
+ * @returns {boolean} true when no CPython environment satisfies the marker
+ */
+export function markerExcludesCPython(marker) {
+  if (typeof marker !== "string" || !marker.trim() || /[()]/.test(marker)) {
+    return false;
+  }
+  return marker
+    .split(/\s+or\s+/)
+    .every((disjunct) =>
+      disjunct.split(/\s+and\s+/).some(implementationClauseIsFalse),
+    );
+}
+
+function recordMarkerEdge(markerEdges, depKey, apkgDep) {
+  if (!depKey) {
+    return;
+  }
+  const edge = markerEdges.get(depKey) || {
+    total: 0,
+    cpythonExcluded: 0,
+    markers: new Set(),
+  };
+  edge.total += 1;
+  const marker = typeof apkgDep === "object" ? apkgDep?.marker : undefined;
+  if (typeof marker === "string" && marker.trim()) {
+    edge.markers.add(marker.trim());
+    if (markerExcludesCPython(marker)) {
+      edge.cpythonExcluded += 1;
+    }
+  }
+  markerEdges.set(depKey, edge);
+}
+
 /**
  * Method to parse python lock files such as poetry.lock, pdm.lock, uv.lock, and pylock.toml.
  *
@@ -911,6 +967,8 @@ export async function parsePyLockData(
   const depsMap = {};
   const existingPkgMap = {};
   const pkgBomRefMap = {};
+  // Incoming edges per dependency, with the environment markers they carry.
+  const markerEdges = new Map();
   let directDepsKeys = {};
   let groupDepsKeys = {};
   let dependencySourceMap = {};
@@ -1080,10 +1138,22 @@ export async function parsePyLockData(
       hasWorkspaces = true;
     }
     for (const amember of workspaceMembers) {
-      if (amember === parentComponent.name) {
+      if (amember === parentComponent?.name) {
+        continue;
+      }
+      // a lock file read without its pyproject.toml has no parent to hold
+      // the members, which is said once for the whole lock file
+      if (!parentComponent) {
+        if (!workspaceWarningShown) {
+          console.log(
+            `The workspace members in ${lockFile || "the lock file"} are read without a pyproject.toml, so there is no project to list them under.`,
+          );
+          workspaceWarningShown = true;
+        }
         continue;
       }
       if (workspaceComponentMap[amember]) {
+        parentComponent.components = parentComponent.components || [];
         parentComponent.components.push(workspaceComponentMap[amember]);
       } else {
         if (!workspaceWarningShown) {
@@ -1334,9 +1404,16 @@ export async function parsePyLockData(
       devDependencies.length ||
       optionalDependencies.length
     ) {
-      if (Array.isArray(apkg.dependencies)) {
+      // A package whose only dependencies are optional (urllib3 has no
+      // required ones, only the `brotli`, `socks` and `zstd` extras) has no
+      // `dependencies` key in uv.lock, but its extras are still edges.
+      if (
+        Array.isArray(apkg.dependencies) ||
+        (!apkg.dependencies &&
+          (devDependencies.length || optionalDependencies.length))
+      ) {
         // pdm.lock files
-        let allDeps = apkg.dependencies;
+        let allDeps = apkg.dependencies || [];
         allDeps = allDeps.concat(devDependencies);
         allDeps = allDeps.concat(optionalDependencies);
         for (const apkgDep of allDeps) {
@@ -1348,6 +1425,7 @@ export async function parsePyLockData(
           const depPkgRef =
             existingPkgMap[nameLower] || existingPkgMap[nameStr];
           depsMap[pkg["bom-ref"]].add(depPkgRef || nameStr);
+          recordMarkerEdge(markerEdges, depPkgRef || nameLower, apkgDep);
           // Propagate the workspace properties to the child components
           if (depPkgRef && pkgBomRefMap[depPkgRef]) {
             const dependentPkg = pkgBomRefMap[depPkgRef];
@@ -1421,6 +1499,28 @@ export async function parsePyLockData(
       }
     }
   }
+  // A package reached only through edges whose environment marker excludes
+  // CPython (brotlicffi is `platform_python_implementation != 'CPython'`) is
+  // not installed by a CPython environment built from this lock, so it is
+  // optional rather than part of the CPython closure. A direct dependency of
+  // the project is left alone: the manifest asked for it.
+  for (const [depRef, edge] of markerEdges) {
+    const dependentPkg = pkgBomRefMap[depRef];
+    if (
+      !dependentPkg ||
+      !edge.total ||
+      edge.cpythonExcluded !== edge.total ||
+      directDepsKeys[normalizePythonDependencyKey(dependentPkg.name)]
+    ) {
+      continue;
+    }
+    dependentPkg.scope = "optional";
+    dependentPkg.properties = dependentPkg.properties || [];
+    dependentPkg.properties.push({
+      name: "cdx:pip:markers",
+      value: Array.from(edge.markers).sort().join(" or "),
+    });
+  }
   // Seed the parent component's first-level edges from the manifest. Lock
   // files describe the resolved closure but do not say which of the locked
   // packages the root project directly depends on; parsePyProjectTomlFile
@@ -1542,14 +1642,40 @@ export async function parsePyLockData(
 }
 
 /**
- * First version clause of a PEP 508 requirement: an operator and the version
- * that follows it.
+ * An exact PEP 440 version clause: `==` or `===` and a version with no
+ * wildcard.
  *
- * `===` (arbitrary equality) is tried before `==`, whitespace may separate the
- * operator from the version, and a version may carry a PEP 440 epoch (`1!2.0`)
- * or a local version label (`2.1.0+cpu`).
+ * Whitespace may separate the operator from the version, and a version may
+ * carry a PEP 440 epoch (`1!2.0`) or a local version label (`2.1.0+cpu`).
  */
-const PEP440_VERSION_CLAUSE = /(===|==|!=|<=|>=|~=|<|>)\s*([0-9A-Za-z.*_+!-]*)/;
+const PEP440_EXACT_CLAUSE = /^(===?)\s*([0-9A-Za-z._+!-]+)$/;
+
+/**
+ * Version a PEP 508 requirement pins: the version of its only exact clause.
+ *
+ * A requirement with no exact clause, a wildcard (`==1.2.*`) or more than one
+ * exact clause pins no version, so a range such as `>=1.0,<2` gives null
+ * rather than one of its bounds. Per-requirement options such as
+ * `--config-settings` are not clauses.
+ *
+ * @param {String} versionSpecifiers Version specifiers of the requirement
+ * @returns {String|null} The pinned version, or null
+ */
+function pinnedRequirementVersion(versionSpecifiers) {
+  const specifiers = versionSpecifiers || "";
+  const optionsIndex = specifiers.indexOf(" --");
+  const clausesSource =
+    optionsIndex === -1 ? specifiers : specifiers.substring(0, optionsIndex);
+  let trimmed = clausesSource.trim();
+  if (trimmed.startsWith("(") && trimmed.endsWith(")")) {
+    trimmed = trimmed.substring(1, trimmed.length - 1);
+  }
+  const exactClauses = trimmed
+    .split(",")
+    .map((clause) => clause.trim().match(PEP440_EXACT_CLAUSE))
+    .filter(Boolean);
+  return exactClauses.length === 1 ? exactClauses[0][2] : null;
+}
 
 /**
  * Method to parse requirements.txt file. This must be replaced with atom parsedeps.
@@ -1738,7 +1864,7 @@ async function parseReqData(
 
     // Handle extras (e.g., package[extra1,extra2])
     let extras = null;
-    const extrasMatch = l.match(/^([a-zA-Z0-9_\-.]+)(\[([^\]]+)])?(.*)$/);
+    const extrasMatch = l.match(/^([a-zA-Z0-9_\-.]+)(\s*\[([^\]]+)])?(.*)$/);
     if (extrasMatch) {
       const [, packageName, , extrasStr, versionSpecifiers] = extrasMatch;
       const name = packageName;
@@ -1749,14 +1875,7 @@ async function parseReqData(
       if (PYTHON_STD_MODULES.includes(name)) {
         continue;
       }
-      const versionMatch = versionSpecifiers.match(PEP440_VERSION_CLAUSE);
-      let version = null;
-      if (versionMatch) {
-        version = versionMatch[2].replaceAll("*", "0") || null;
-        if (version === "0") {
-          version = null;
-        }
-      }
+      const version = pinnedRequirementVersion(versionSpecifiers);
       const apkg = {
         name,
         version,
@@ -1776,7 +1895,10 @@ async function parseReqData(
           value: extras.join(","),
         });
       }
-      if (versionSpecifiers && !versionSpecifiers.trim().startsWith("==")) {
+      if (
+        versionSpecifiers?.trim() &&
+        (!version || !versionSpecifiers.trim().startsWith("=="))
+      ) {
         properties.push({
           name: "cdx:pypi:versionSpecifiers",
           value: versionSpecifiers.trim(),
@@ -1813,12 +1935,7 @@ async function parseReqData(
       if (PYTHON_STD_MODULES.includes(name)) {
         continue;
       }
-      const versionMatch = versionSpecifiers.match(PEP440_VERSION_CLAUSE);
-      let version = null;
-      if (versionMatch) {
-        version = versionMatch[2].replaceAll("*", "0") || null;
-        if (version === "0") version = null;
-      }
+      const version = pinnedRequirementVersion(versionSpecifiers);
       const apkg = {
         name,
         version,
@@ -1829,7 +1946,10 @@ async function parseReqData(
       if (licenses) {
         apkg.licenses = licenses;
       }
-      if (versionSpecifiers && !versionSpecifiers.trim().startsWith("==")) {
+      if (
+        versionSpecifiers?.trim() &&
+        (!version || !versionSpecifiers.trim().startsWith("=="))
+      ) {
         properties.push({
           name: "cdx:pypi:versionSpecifiers",
           value: versionSpecifiers.trim(),
@@ -1977,6 +2097,33 @@ function createPixiBomRef(packageData) {
   return `library:${packageData["name"]}:${packageData["version"] || ""}`;
 }
 
+/**
+ * Return the download url of a pixi.lock package, without credentials, query
+ * or fragment. A private conda channel can carry its access token in a
+ * `/t/<token>/` path segment, so such a url is left out.
+ *
+ * @param {unknown} url Package url from pixi.lock
+ * @returns {string|undefined} Distribution url
+ */
+function pixiDistributionUrl(url) {
+  if (typeof url !== "string" || !url.trim()) {
+    return undefined;
+  }
+  const sanitized = sanitizeBomUrl(url);
+  try {
+    const parsed = new URL(sanitized);
+    if (
+      !["http:", "https:"].includes(parsed.protocol) ||
+      parsed.pathname.split("/").includes("t")
+    ) {
+      return undefined;
+    }
+  } catch {
+    return undefined;
+  }
+  return sanitized;
+}
+
 /**
  * Method to parse pixi.lock data
  *
@@ -2066,10 +2213,13 @@ export function parsePixiLockFile(pixiLockFileName, path) {
      *
      */
     const purlTemplate = createPurlTemplate(packageData);
+    const distributionUrl = pixiDistributionUrl(packageData["url"]);
     return {
       name: packageData["name"],
       version: packageData["version"],
       ...(purlTemplate ? { purl: purlTemplate } : {}),
+      // The package download url becomes a distribution external reference.
+      ...(distributionUrl ? { distribution: { url: distributionUrl } } : {}),
       type: "library",
       "bom-ref": createPixiBomRef(packageData),
       // "licenses": [
@@ -2077,10 +2227,8 @@ export function parsePixiLockFile(pixiLockFileName, path) {
       //       "id": packageData["license"]
       //   }]
       // ],
-      supplier: {
-        name: packageData["build"],
-        url: packageData["url"],
-      },
+      // The conda build string is not a supplier, so it stays in
+      // cdx:pixi:build below.
       // "hashes": [
       //   {"md5": packageData["md5"]},
       //   {"sha256": packageData["sha256"]}