@cdxgen/cdxgen 13.5.0 → 13.6.0

package/lib/ecosystems/parsers-misc.js

@@ -1,5 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { readFileSync } from "node:fs";
+import { readdirSync, readFileSync } from "node:fs";
 import {
   sep as _sep,
   basename,
@@ -18,6 +18,7 @@ import {
   hasDangerousUnicode,
   readEnvironmentVariable,
 } from "../core/activity.js";
+import { readRegularTextFile } from "../core/boundedFile.js";
 import { recordDegradation } from "../core/buildLedger.js";
 import { deferFailOnError } from "../core/deferredExit.js";
 import { shouldFetchLicense } from "../core/env.js";
@@ -52,7 +53,12 @@ import {
   tryBuildPurl,
 } from "../inventory/purl.js";
 import { getLicenses, spdxLicenses } from "../inventory/spdx.js";
-import { cmakeFetchDependencies } from "../parsers/cmakeLists.js";
+import {
+  cmakeFetchDependencies,
+  cmakeSetVariables,
+  expandCmakeVariables,
+  parseCmakeCommands,
+} from "../parsers/cmakeLists.js";
 import { isValidIdnEmail } from "../parsers/email.js";
 import { submodulePurlCoordinates } from "../parsers/gitmodules.js";
 import { getDartMetadata } from "./ecosystems.js";
@@ -2784,6 +2790,130 @@ export function cmakeFetchDependencyComponent(dep, srcFile) {
   return component;
 }
 
+const MAX_MESON_WRAPS = 256;
+const MAX_MESON_WRAP_BYTES = 64 * 1024;
+
+function upstreamVersionOf(text) {
+  if (typeof text !== "string") {
+    return undefined;
+  }
+  const value = text.trim().replace(/^v(?=\d)/, "");
+  return /^\d+(\.\d+)*([-+._~][0-9A-Za-z.]+)?$/.test(value) ? value : undefined;
+}
+
+// `brotli-1.1.0`, `brotli-1.1.0.tar.gz`: the version after the last `-` that
+// starts a number.
+function versionFromArchiveName(name) {
+  if (typeof name !== "string") {
+    return undefined;
+  }
+  const stem = name.replace(/\.(tar\.gz|tar\.xz|tar\.bz2|tgz|zip|tar)$/i, "");
+  const dash = stem.search(/-\d[^-]*$/);
+  return dash === -1 ? undefined : upstreamVersionOf(stem.slice(dash + 1));
+}
+
+/**
+ * Read the wrap files of a meson project's subprojects directory: which
+ * dependency names each wrap provides, and the version it pins.
+ * https://mesonbuild.com/Wrap-dependency-system-manual.html
+ *
+ * A WrapDB wrap records `wrapdb_version = <upstream>-<wrap revision>`; any wrap
+ * names its unpacked `directory` and its archive, which carry the upstream
+ * version; a git wrap pins a `revision` that is often a version tag.
+ *
+ * @param {string} subprojectsDir path of the subprojects directory
+ * @returns {Map<string, {wrap: string, version: string, source: string, revision?: string}>}
+ *   by provided dependency name, lower-case
+ */
+export function readMesonWraps(subprojectsDir) {
+  const provided = new Map();
+  let wrapFiles = [];
+  try {
+    wrapFiles = readdirSync(subprojectsDir)
+      .filter((name) => name.endsWith(".wrap"))
+      .sort()
+      .slice(0, MAX_MESON_WRAPS);
+  } catch {
+    return provided;
+  }
+  for (const wrapFile of wrapFiles) {
+    // A regular file only: a FIFO named *.wrap would block a plain read.
+    const text = readRegularTextFile(
+      join(subprojectsDir, wrapFile),
+      MAX_MESON_WRAP_BYTES,
+    );
+    if (text === undefined) {
+      continue;
+    }
+    const sections = {};
+    let section = "";
+    for (const rawLine of text.split(/\r?\n/)) {
+      const line = rawLine.trim();
+      if (!line || line.startsWith("#") || line.startsWith(";")) {
+        continue;
+      }
+      if (line.startsWith("[") && line.endsWith("]")) {
+        section = line.slice(1, -1).trim();
+        sections[section] = sections[section] || {};
+        continue;
+      }
+      const equals = line.indexOf("=");
+      if (equals <= 0 || !section) {
+        continue;
+      }
+      sections[section][line.slice(0, equals).trim()] = line
+        .slice(equals + 1)
+        .trim();
+    }
+    const body = sections["wrap-file"] || sections["wrap-git"] || {};
+    let version;
+    let source;
+    let revision;
+    if (body.wrapdb_version) {
+      const dash = body.wrapdb_version.lastIndexOf("-");
+      version = upstreamVersionOf(
+        dash > 0 ? body.wrapdb_version.slice(0, dash) : body.wrapdb_version,
+      );
+      revision = dash > 0 ? body.wrapdb_version.slice(dash + 1) : undefined;
+      source = "wrapdb_version";
+    }
+    if (!version && sections["wrap-file"]) {
+      version =
+        versionFromArchiveName(body.directory) ||
+        versionFromArchiveName(body.source_filename);
+      source = version ? "directory" : undefined;
+    }
+    if (!version && sections["wrap-git"]) {
+      version = upstreamVersionOf(body.revision);
+      source = version ? "revision" : undefined;
+    }
+    if (!version) {
+      continue;
+    }
+    const wrap = wrapFile.slice(0, -".wrap".length);
+    const entry = { wrap, version, source, ...(revision ? { revision } : {}) };
+    const names = new Set([wrap.toLowerCase()]);
+    const provide = sections.provide || {};
+    for (const [key, value] of Object.entries(provide)) {
+      if (key === "dependency_names" || key === "program_names") {
+        for (const name of value.split(",")) {
+          if (name.trim()) {
+            names.add(name.trim().toLowerCase());
+          }
+        }
+      } else {
+        names.add(key.toLowerCase());
+      }
+    }
+    for (const name of names) {
+      if (!provided.has(name)) {
+        provided.set(name, entry);
+      }
+    }
+  }
+  return provided;
+}
+
 /**
  * Parse a CMake-like build file (CMakeLists.txt, meson.build, etc.) and extract the
  * parent component and list of dependency packages.
@@ -2801,6 +2931,11 @@ export function cmakeFetchDependencyComponent(dep, srcFile) {
  */
 export function parseCmakeLikeFile(cmakeListFile, pkgType, options = {}) {
   const rawData = readFileSync(cmakeListFile, { encoding: "utf-8" });
+  // The wraps a meson project ships pin the versions of the subprojects its
+  // dependency() calls fall back to.
+  const mesonWraps = basename(cmakeListFile).startsWith("meson")
+    ? readMesonWraps(join(dirname(cmakeListFile), "subprojects"))
+    : new Map();
   let cmakeListData = rawData;
   const pkgList = [];
   const pkgAddedMap = {};
@@ -2808,6 +2943,25 @@ export function parseCmakeLikeFile(cmakeListFile, pkgType, options = {}) {
   const versionsMap = {};
   let parentComponent = {};
   const templateValues = {};
+  // The parent component is built the same way whether its name and version
+  // came from the parsed project() command or from the line loop below.
+  const toParentComponent = (parentName, parentVersion) => {
+    const component = {
+      group: options.projectGroup || "",
+      name: parentName,
+      version: parentVersion || options.projectVersion || "",
+      type: "application",
+    };
+    component["purl"] = build({
+      type: pkgType,
+      namespace: component.group || null,
+      name: component.name,
+      version: component.version || null,
+      subpath: null,
+    });
+    component["bom-ref"] = decodeURIComponent(component["purl"]);
+    return component;
+  };
   // A dependency fetched at configure time is the same package a find_package()
   // of the same name asks for: the fetched component, with its source and pin,
   // stands for both.
@@ -2823,6 +2977,38 @@ export function parseCmakeLikeFile(cmakeListFile, pkgType, options = {}) {
       fetchedNames.add(dep.name.toLowerCase());
       pkgList.push(component);
     }
+    // A project() call may span several lines, which the line loop below
+    // cannot rejoin, so the parent name and version are read from the parsed
+    // commands instead. The first argument is the project name and the
+    // VERSION keyword is followed by the version.
+    const commands = parseCmakeCommands(rawData);
+    const variables = cmakeSetVariables(commands);
+    const projectCommand = commands.find(
+      (command) => command.name === "project",
+    );
+    if (projectCommand?.args.length) {
+      const args = projectCommand.args.map((arg) =>
+        expandCmakeVariables(arg, variables),
+      );
+      const parentName = args[0].replace(":", "").trim();
+      let parentVersion;
+      const versionIndex = args.findIndex(
+        (arg) => arg.toLowerCase() === "version",
+      );
+      if (versionIndex > -1 && args.length > versionIndex + 1) {
+        parentVersion = args[versionIndex + 1];
+      }
+      // A `${VAR}` that no `set()` in this file defines stays literal. It is
+      // not a version, and percent-encoding it into a purl publishes the
+      // variable name as the version, so the field is left empty instead.
+      // CMakeCache.txt carries the evaluated value when a build tree exists.
+      if (parentVersion?.includes("$")) {
+        parentVersion = undefined;
+      }
+      if (parentName.length && !parentName.includes("$")) {
+        parentComponent = toParentComponent(parentName, parentVersion);
+      }
+    }
   }
   cmakeListData = cmakeListData
     .replace(/^ {2}/g, "")
@@ -2883,23 +3069,7 @@ export function parseCmakeLikeFile(cmakeListFile, pkgType, options = {}) {
         parentVersion = undefined;
       }
       if (parentName?.length && !parentName.includes("$")) {
-        parentComponent = {
-          group: options.projectGroup || "",
-          name: parentName,
-          version: parentVersion || options.projectVersion || "",
-          type: "application",
-        };
-        const safePath = path ? path.replace(/^\/+/, "") : null;
-        parentComponent["purl"] = build({
-          type: pkgType,
-          namespace: parentComponent.group || null,
-          name: parentComponent.name,
-          version: parentComponent.version || null,
-          subpath: safePath,
-        });
-        parentComponent["bom-ref"] = decodeURIComponent(
-          parentComponent["purl"],
-        );
+        parentComponent = toParentComponent(parentName, parentVersion);
       }
     } else if (CMAKE_FIND_COMMANDS.has(command)) {
       let tmpB = commandArgs.split(")")[0].split(" ");
@@ -3041,7 +3211,26 @@ export function parseCmakeLikeFile(cmakeListFile, pkgType, options = {}) {
         // wrap match has since replaced, so the scraped name is the fallback.
         const versionSpecifier =
           versionSpecifiersMap[n] || versionSpecifiersMap[scrapedName];
-        const version = versionsMap[n] || versionsMap[scrapedName] || "";
+        let version = versionsMap[n] || versionsMap[scrapedName] || "";
+        const localWrap =
+          mesonWraps.get(scrapedName.toLowerCase()) ||
+          mesonWraps.get(n.toLowerCase());
+        if (localWrap && !version) {
+          version = localWrap.version;
+          props.push({ name: "cdx:meson:wrap", value: localWrap.wrap });
+          props.push({
+            name: "cdx:meson:wrap:versionSource",
+            value: localWrap.source,
+          });
+          if (localWrap.revision) {
+            props.push({
+              name: "cdx:meson:wrap:revision",
+              value: localWrap.revision,
+            });
+          }
+          // A pinned wrap is a stronger identity than a name match alone.
+          confidence = Math.max(confidence, 0.7);
+        }
         if (versionSpecifier) {
           props.push({
             name: "cdx:build:versionSpecifiers",

package/lib/ecosystems/parsers-python.js

@@ -888,6 +888,61 @@ export function enrichFromInstalledMetadata(pkgList, installedMetadata) {
   return enriched;
 }
 
+// The interpreter cdxgen assumes for a lock file's environment. A marker that
+// tests the implementation against it is decidable without a target machine.
+const IMPLEMENTATION_CLAUSE_RE =
+  /^(platform_python_implementation|implementation_name)\s*(==|!=)\s*(['"])([A-Za-z]+)\3$/;
+
+function implementationClauseIsFalse(clause) {
+  const match = IMPLEMENTATION_CLAUSE_RE.exec(clause.trim());
+  if (!match) {
+    return false;
+  }
+  const expected = match[1] === "implementation_name" ? "cpython" : "CPython";
+  const equal = match[4] === expected;
+  return match[2] === "==" ? !equal : equal;
+}
+
+/**
+ * Whether an environment marker is false for every CPython interpreter. Only
+ * `platform_python_implementation` and `implementation_name` clauses are
+ * decided; anything else, or a marker with parentheses, is treated as possibly
+ * true, so the answer errs towards keeping a package.
+ *
+ * @param {string} marker PEP 508 environment marker
+ * @returns {boolean} true when no CPython environment satisfies the marker
+ */
+export function markerExcludesCPython(marker) {
+  if (typeof marker !== "string" || !marker.trim() || /[()]/.test(marker)) {
+    return false;
+  }
+  return marker
+    .split(/\s+or\s+/)
+    .every((disjunct) =>
+      disjunct.split(/\s+and\s+/).some(implementationClauseIsFalse),
+    );
+}
+
+function recordMarkerEdge(markerEdges, depKey, apkgDep) {
+  if (!depKey) {
+    return;
+  }
+  const edge = markerEdges.get(depKey) || {
+    total: 0,
+    cpythonExcluded: 0,
+    markers: new Set(),
+  };
+  edge.total += 1;
+  const marker = typeof apkgDep === "object" ? apkgDep?.marker : undefined;
+  if (typeof marker === "string" && marker.trim()) {
+    edge.markers.add(marker.trim());
+    if (markerExcludesCPython(marker)) {
+      edge.cpythonExcluded += 1;
+    }
+  }
+  markerEdges.set(depKey, edge);
+}
+
 /**
  * Method to parse python lock files such as poetry.lock, pdm.lock, uv.lock, and pylock.toml.
  *
@@ -911,6 +966,8 @@ export async function parsePyLockData(
   const depsMap = {};
   const existingPkgMap = {};
   const pkgBomRefMap = {};
+  // Incoming edges per dependency, with the environment markers they carry.
+  const markerEdges = new Map();
   let directDepsKeys = {};
   let groupDepsKeys = {};
   let dependencySourceMap = {};
@@ -1334,9 +1391,16 @@ export async function parsePyLockData(
       devDependencies.length ||
       optionalDependencies.length
     ) {
-      if (Array.isArray(apkg.dependencies)) {
+      // A package whose only dependencies are optional (urllib3 has no
+      // required ones, only the `brotli`, `socks` and `zstd` extras) has no
+      // `dependencies` key in uv.lock, but its extras are still edges.
+      if (
+        Array.isArray(apkg.dependencies) ||
+        (!apkg.dependencies &&
+          (devDependencies.length || optionalDependencies.length))
+      ) {
         // pdm.lock files
-        let allDeps = apkg.dependencies;
+        let allDeps = apkg.dependencies || [];
         allDeps = allDeps.concat(devDependencies);
         allDeps = allDeps.concat(optionalDependencies);
         for (const apkgDep of allDeps) {
@@ -1348,6 +1412,7 @@ export async function parsePyLockData(
           const depPkgRef =
             existingPkgMap[nameLower] || existingPkgMap[nameStr];
           depsMap[pkg["bom-ref"]].add(depPkgRef || nameStr);
+          recordMarkerEdge(markerEdges, depPkgRef || nameLower, apkgDep);
           // Propagate the workspace properties to the child components
           if (depPkgRef && pkgBomRefMap[depPkgRef]) {
             const dependentPkg = pkgBomRefMap[depPkgRef];
@@ -1421,6 +1486,28 @@ export async function parsePyLockData(
       }
     }
   }
+  // A package reached only through edges whose environment marker excludes
+  // CPython (brotlicffi is `platform_python_implementation != 'CPython'`) is
+  // not installed by a CPython environment built from this lock, so it is
+  // optional rather than part of the CPython closure. A direct dependency of
+  // the project is left alone: the manifest asked for it.
+  for (const [depRef, edge] of markerEdges) {
+    const dependentPkg = pkgBomRefMap[depRef];
+    if (
+      !dependentPkg ||
+      !edge.total ||
+      edge.cpythonExcluded !== edge.total ||
+      directDepsKeys[normalizePythonDependencyKey(dependentPkg.name)]
+    ) {
+      continue;
+    }
+    dependentPkg.scope = "optional";
+    dependentPkg.properties = dependentPkg.properties || [];
+    dependentPkg.properties.push({
+      name: "cdx:pip:markers",
+      value: Array.from(edge.markers).sort().join(" or "),
+    });
+  }
   // Seed the parent component's first-level edges from the manifest. Lock
   // files describe the resolved closure but do not say which of the locked
   // packages the root project directly depends on; parsePyProjectTomlFile

package/lib/ecosystems/pythonutils.js

@@ -11,6 +11,7 @@ import {
 import process from "node:process";
 
 import { build } from "@cdxgen/cdx-purl";
+import { coerce, gte } from "semver";
 
 import {
   DEBUG_MODE,
@@ -25,6 +26,7 @@ import {
   PYPI_MODULE_PACKAGE_MAPPING,
   PYTHON_STD_MODULES,
 } from "../core/state.js";
+import { parseToml } from "../core/toml.js";
 import { findAppModules } from "../inventory/atomUtils.js";
 import { flattenDeps } from "../inventory/deps.js";
 import { pypiBomRef } from "../inventory/purl.js";
@@ -612,6 +614,128 @@ export function createUVLock(basePath, options) {
   }
 }
 
+const CONSTRAINT_OPERATORS = [">=", "<=", "==", "!=", "~=", ">", "<", "^", "="];
+const LOWER_BOUND_OPERATORS = new Set([">=", ">", "==", "~=", "^", "="]);
+
+/**
+ * Check whether a single version constraint puts the lower bound at version 2
+ * or higher.
+ *
+ * @param {string} constraint Constraint such as ">=2.0.0" or "<3.0.0"
+ * @returns {boolean} true when the constraint requires version 2 or newer
+ */
+function requiresVersion2(constraint) {
+  let operator = "";
+  let version = constraint.trim();
+  for (const candidate of CONSTRAINT_OPERATORS) {
+    if (version.startsWith(candidate)) {
+      operator = candidate;
+      version = version.slice(candidate.length).trim();
+      break;
+    }
+  }
+  if (!operator || !LOWER_BOUND_OPERATORS.has(operator)) {
+    return false;
+  }
+  const coerced = coerce(version);
+  return coerced !== null && gte(coerced, "2.0.0");
+}
+
+/**
+ * Check whether a build requirement names poetry-core with a lower bound of
+ * version 2 or higher.
+ *
+ * @param {string} requirement Build requirement such as "poetry-core>=2.0.0"
+ * @returns {boolean} true when poetry-core 2 or newer is required
+ */
+function requiresPoetryCore2(requirement) {
+  if (typeof requirement !== "string") {
+    return false;
+  }
+  const entry = requirement.trim().toLowerCase();
+  if (!entry.startsWith("poetry-core")) {
+    return false;
+  }
+  let spec = entry.slice("poetry-core".length);
+  if (spec.startsWith("[")) {
+    const extrasEnd = spec.indexOf("]");
+    if (extrasEnd === -1) {
+      return false;
+    }
+    spec = spec.slice(extrasEnd + 1);
+  }
+  const next = spec.charAt(0);
+  if (next && next !== " " && next !== "\t" && !"><=!~^".includes(next)) {
+    return false;
+  }
+  return spec
+    .trim()
+    .split(",")
+    .some((constraint) => requiresVersion2(constraint));
+}
+
+/**
+ * Check whether a pyproject.toml requires poetry-core 2 or newer to build.
+ *
+ * @param {string} pyprojectFile Path to pyproject.toml
+ * @returns {boolean} true when the project needs Poetry 2 or newer
+ */
+export function isPoetry2Project(pyprojectFile) {
+  if (!pyprojectFile || !safeExistsSync(pyprojectFile)) {
+    return false;
+  }
+  let config;
+  try {
+    config = parseToml(readFileSync(pyprojectFile, "utf-8"));
+  } catch (_err) {
+    return false;
+  }
+  const requires = config?.["build-system"]?.requires;
+  if (!Array.isArray(requires)) {
+    return false;
+  }
+  return requires.some((requirement) => requiresPoetryCore2(requirement));
+}
+
+/**
+ * Find the pyproject.toml belonging to a poetry trigger file.
+ *
+ * @param {string} triggerFile pyproject.toml or poetry.lock path
+ * @returns {string|null} Path to pyproject.toml, or null when there is none
+ */
+function findPoetryPyprojectFile(triggerFile) {
+  if (!triggerFile) {
+    return null;
+  }
+  if (triggerFile.endsWith("pyproject.toml")) {
+    return triggerFile;
+  }
+  if (triggerFile.endsWith("poetry.lock")) {
+    const sibling = join(dirname(triggerFile), "pyproject.toml");
+    return safeExistsSync(sibling) ? sibling : null;
+  }
+  return null;
+}
+
+/**
+ * Build the poetry install arguments for a poetry project.
+ *
+ * Poetry 2 installs only the main dependency group and no extras by default,
+ * so projects requiring poetry-core 2 or newer also get --all-groups and
+ * --all-extras. Callers prepend the python invocation such as -m poetry.
+ *
+ * @param {string} pyprojectFile pyproject.toml or poetry.lock path of the project
+ * @returns {string[]} poetry install arguments
+ */
+export function getPoetryInstallArgs(pyprojectFile) {
+  const args = ["install", "-n", "--no-root"];
+  const resolvedFile = findPoetryPyprojectFile(pyprojectFile);
+  if (resolvedFile && isPoetry2Project(resolvedFile)) {
+    args.push("--all-groups", "--all-extras");
+  }
+  return args;
+}
+
 /**
  * Execute pip freeze by creating a virtual env in a temp directory and construct the dependency tree
  *
@@ -742,7 +866,9 @@ export async function getPipFrozenTree(
         shell: isWin,
       });
       thoughtLog("Performing poetry install");
-      let poetryInstallArgs = ["-m", "poetry", "install", "-n", "--no-root"];
+      let poetryInstallArgs = ["-m", "poetry"].concat(
+        getPoetryInstallArgs(reqOrSetupFile),
+      );
       if (isSecureMode) {
         poetryInstallArgs.unshift("-S");
       }
@@ -756,7 +882,7 @@ export async function getPipFrozenTree(
           thoughtLog(
             "Hmm, poetry doesn't seem to be available as a module. Perhaps it was installed directly 🤔?",
           );
-          poetryInstallArgs = ["install", "-n", "--no-root"];
+          poetryInstallArgs = getPoetryInstallArgs(reqOrSetupFile);
           // Attempt to perform poetry install
           result = safeSpawnSync("poetry", poetryInstallArgs, {
             cwd: basePath,