@cdxgen/cdxgen 13.4.0 → 13.6.0

package/lib/ecosystems/parsers-python.js

@@ -1,6 +1,5 @@
 import { readFileSync } from "node:fs";
 import { basename, dirname, join, relative } from "node:path";
-import process from "node:process";
 import { URL } from "node:url";
 
 import { build } from "@cdxgen/cdx-purl";
@@ -889,6 +888,61 @@ export function enrichFromInstalledMetadata(pkgList, installedMetadata) {
   return enriched;
 }
 
+// The interpreter cdxgen assumes for a lock file's environment. A marker that
+// tests the implementation against it is decidable without a target machine.
+const IMPLEMENTATION_CLAUSE_RE =
+  /^(platform_python_implementation|implementation_name)\s*(==|!=)\s*(['"])([A-Za-z]+)\3$/;
+
+function implementationClauseIsFalse(clause) {
+  const match = IMPLEMENTATION_CLAUSE_RE.exec(clause.trim());
+  if (!match) {
+    return false;
+  }
+  const expected = match[1] === "implementation_name" ? "cpython" : "CPython";
+  const equal = match[4] === expected;
+  return match[2] === "==" ? !equal : equal;
+}
+
+/**
+ * Whether an environment marker is false for every CPython interpreter. Only
+ * `platform_python_implementation` and `implementation_name` clauses are
+ * decided; anything else, or a marker with parentheses, is treated as possibly
+ * true, so the answer errs towards keeping a package.
+ *
+ * @param {string} marker PEP 508 environment marker
+ * @returns {boolean} true when no CPython environment satisfies the marker
+ */
+export function markerExcludesCPython(marker) {
+  if (typeof marker !== "string" || !marker.trim() || /[()]/.test(marker)) {
+    return false;
+  }
+  return marker
+    .split(/\s+or\s+/)
+    .every((disjunct) =>
+      disjunct.split(/\s+and\s+/).some(implementationClauseIsFalse),
+    );
+}
+
+function recordMarkerEdge(markerEdges, depKey, apkgDep) {
+  if (!depKey) {
+    return;
+  }
+  const edge = markerEdges.get(depKey) || {
+    total: 0,
+    cpythonExcluded: 0,
+    markers: new Set(),
+  };
+  edge.total += 1;
+  const marker = typeof apkgDep === "object" ? apkgDep?.marker : undefined;
+  if (typeof marker === "string" && marker.trim()) {
+    edge.markers.add(marker.trim());
+    if (markerExcludesCPython(marker)) {
+      edge.cpythonExcluded += 1;
+    }
+  }
+  markerEdges.set(depKey, edge);
+}
+
 /**
  * Method to parse python lock files such as poetry.lock, pdm.lock, uv.lock, and pylock.toml.
  *
@@ -912,6 +966,8 @@ export async function parsePyLockData(
   const depsMap = {};
   const existingPkgMap = {};
   const pkgBomRefMap = {};
+  // Incoming edges per dependency, with the environment markers they carry.
+  const markerEdges = new Map();
   let directDepsKeys = {};
   let groupDepsKeys = {};
   let dependencySourceMap = {};
@@ -1335,9 +1391,16 @@ export async function parsePyLockData(
       devDependencies.length ||
       optionalDependencies.length
     ) {
-      if (Array.isArray(apkg.dependencies)) {
+      // A package whose only dependencies are optional (urllib3 has no
+      // required ones, only the `brotli`, `socks` and `zstd` extras) has no
+      // `dependencies` key in uv.lock, but its extras are still edges.
+      if (
+        Array.isArray(apkg.dependencies) ||
+        (!apkg.dependencies &&
+          (devDependencies.length || optionalDependencies.length))
+      ) {
         // pdm.lock files
-        let allDeps = apkg.dependencies;
+        let allDeps = apkg.dependencies || [];
         allDeps = allDeps.concat(devDependencies);
         allDeps = allDeps.concat(optionalDependencies);
         for (const apkgDep of allDeps) {
@@ -1349,6 +1412,7 @@ export async function parsePyLockData(
           const depPkgRef =
             existingPkgMap[nameLower] || existingPkgMap[nameStr];
           depsMap[pkg["bom-ref"]].add(depPkgRef || nameStr);
+          recordMarkerEdge(markerEdges, depPkgRef || nameLower, apkgDep);
           // Propagate the workspace properties to the child components
           if (depPkgRef && pkgBomRefMap[depPkgRef]) {
             const dependentPkg = pkgBomRefMap[depPkgRef];
@@ -1422,6 +1486,28 @@ export async function parsePyLockData(
       }
     }
   }
+  // A package reached only through edges whose environment marker excludes
+  // CPython (brotlicffi is `platform_python_implementation != 'CPython'`) is
+  // not installed by a CPython environment built from this lock, so it is
+  // optional rather than part of the CPython closure. A direct dependency of
+  // the project is left alone: the manifest asked for it.
+  for (const [depRef, edge] of markerEdges) {
+    const dependentPkg = pkgBomRefMap[depRef];
+    if (
+      !dependentPkg ||
+      !edge.total ||
+      edge.cpythonExcluded !== edge.total ||
+      directDepsKeys[normalizePythonDependencyKey(dependentPkg.name)]
+    ) {
+      continue;
+    }
+    dependentPkg.scope = "optional";
+    dependentPkg.properties = dependentPkg.properties || [];
+    dependentPkg.properties.push({
+      name: "cdx:pip:markers",
+      value: Array.from(edge.markers).sort().join(" or "),
+    });
+  }
   // Seed the parent component's first-level edges from the manifest. Lock
   // files describe the resolved closure but do not say which of the locked
   // packages the root project directly depends on; parsePyProjectTomlFile
@@ -2205,28 +2291,30 @@ export function parsePixiTomlFile(pixiToml) {
 }
 
 /**
- * Method to run cli command `pixi install`
- *
+ * Method to run cli command `pixi install` in the project directory.
  *
+ * @param {string} projectPath Directory containing pixi.toml
+ * @returns {boolean} true when pixi install succeeded
  */
-export function generatePixiLockFile(_path) {
-  const result = safeSpawnSync("pixi", ["install"]);
+export function generatePixiLockFile(projectPath) {
+  const result = safeSpawnSync("pixi", ["install"], { cwd: projectPath });
 
   if (result.status !== 0) {
-    // Handle errors
+    // Handle errors. The caller carries on without a lock file: exiting here
+    // would also stop a long-running cdxgen server.
     if (result.error && result.error.code === "ENOENT") {
       console.error(
         "Error: pixi command not found. Make sure pixi.js is installed globally.",
       );
     } else {
       console.error(
-        `Error executing pixi install: ${result.error || result.stderr.toString()}`,
+        `Error executing pixi install: ${result.error || result.stderr?.toString()}`,
       );
     }
-    process.exit(1);
-  } else {
-    console.log("Dependencies installed successfully.");
+    return false;
   }
+  console.log("Dependencies installed successfully.");
+  return true;
 }
 
 /**

package/lib/ecosystems/pythonutils.js

@@ -11,6 +11,7 @@ import {
 import process from "node:process";
 
 import { build } from "@cdxgen/cdx-purl";
+import { coerce, gte } from "semver";
 
 import {
   DEBUG_MODE,
@@ -25,6 +26,7 @@ import {
   PYPI_MODULE_PACKAGE_MAPPING,
   PYTHON_STD_MODULES,
 } from "../core/state.js";
+import { parseToml } from "../core/toml.js";
 import { findAppModules } from "../inventory/atomUtils.js";
 import { flattenDeps } from "../inventory/deps.js";
 import { pypiBomRef } from "../inventory/purl.js";
@@ -612,6 +614,128 @@ export function createUVLock(basePath, options) {
   }
 }
 
+const CONSTRAINT_OPERATORS = [">=", "<=", "==", "!=", "~=", ">", "<", "^", "="];
+const LOWER_BOUND_OPERATORS = new Set([">=", ">", "==", "~=", "^", "="]);
+
+/**
+ * Check whether a single version constraint puts the lower bound at version 2
+ * or higher.
+ *
+ * @param {string} constraint Constraint such as ">=2.0.0" or "<3.0.0"
+ * @returns {boolean} true when the constraint requires version 2 or newer
+ */
+function requiresVersion2(constraint) {
+  let operator = "";
+  let version = constraint.trim();
+  for (const candidate of CONSTRAINT_OPERATORS) {
+    if (version.startsWith(candidate)) {
+      operator = candidate;
+      version = version.slice(candidate.length).trim();
+      break;
+    }
+  }
+  if (!operator || !LOWER_BOUND_OPERATORS.has(operator)) {
+    return false;
+  }
+  const coerced = coerce(version);
+  return coerced !== null && gte(coerced, "2.0.0");
+}
+
+/**
+ * Check whether a build requirement names poetry-core with a lower bound of
+ * version 2 or higher.
+ *
+ * @param {string} requirement Build requirement such as "poetry-core>=2.0.0"
+ * @returns {boolean} true when poetry-core 2 or newer is required
+ */
+function requiresPoetryCore2(requirement) {
+  if (typeof requirement !== "string") {
+    return false;
+  }
+  const entry = requirement.trim().toLowerCase();
+  if (!entry.startsWith("poetry-core")) {
+    return false;
+  }
+  let spec = entry.slice("poetry-core".length);
+  if (spec.startsWith("[")) {
+    const extrasEnd = spec.indexOf("]");
+    if (extrasEnd === -1) {
+      return false;
+    }
+    spec = spec.slice(extrasEnd + 1);
+  }
+  const next = spec.charAt(0);
+  if (next && next !== " " && next !== "\t" && !"><=!~^".includes(next)) {
+    return false;
+  }
+  return spec
+    .trim()
+    .split(",")
+    .some((constraint) => requiresVersion2(constraint));
+}
+
+/**
+ * Check whether a pyproject.toml requires poetry-core 2 or newer to build.
+ *
+ * @param {string} pyprojectFile Path to pyproject.toml
+ * @returns {boolean} true when the project needs Poetry 2 or newer
+ */
+export function isPoetry2Project(pyprojectFile) {
+  if (!pyprojectFile || !safeExistsSync(pyprojectFile)) {
+    return false;
+  }
+  let config;
+  try {
+    config = parseToml(readFileSync(pyprojectFile, "utf-8"));
+  } catch (_err) {
+    return false;
+  }
+  const requires = config?.["build-system"]?.requires;
+  if (!Array.isArray(requires)) {
+    return false;
+  }
+  return requires.some((requirement) => requiresPoetryCore2(requirement));
+}
+
+/**
+ * Find the pyproject.toml belonging to a poetry trigger file.
+ *
+ * @param {string} triggerFile pyproject.toml or poetry.lock path
+ * @returns {string|null} Path to pyproject.toml, or null when there is none
+ */
+function findPoetryPyprojectFile(triggerFile) {
+  if (!triggerFile) {
+    return null;
+  }
+  if (triggerFile.endsWith("pyproject.toml")) {
+    return triggerFile;
+  }
+  if (triggerFile.endsWith("poetry.lock")) {
+    const sibling = join(dirname(triggerFile), "pyproject.toml");
+    return safeExistsSync(sibling) ? sibling : null;
+  }
+  return null;
+}
+
+/**
+ * Build the poetry install arguments for a poetry project.
+ *
+ * Poetry 2 installs only the main dependency group and no extras by default,
+ * so projects requiring poetry-core 2 or newer also get --all-groups and
+ * --all-extras. Callers prepend the python invocation such as -m poetry.
+ *
+ * @param {string} pyprojectFile pyproject.toml or poetry.lock path of the project
+ * @returns {string[]} poetry install arguments
+ */
+export function getPoetryInstallArgs(pyprojectFile) {
+  const args = ["install", "-n", "--no-root"];
+  const resolvedFile = findPoetryPyprojectFile(pyprojectFile);
+  if (resolvedFile && isPoetry2Project(resolvedFile)) {
+    args.push("--all-groups", "--all-extras");
+  }
+  return args;
+}
+
 /**
  * Execute pip freeze by creating a virtual env in a temp directory and construct the dependency tree
  *
@@ -742,7 +866,9 @@ export async function getPipFrozenTree(
         shell: isWin,
       });
       thoughtLog("Performing poetry install");
-      let poetryInstallArgs = ["-m", "poetry", "install", "-n", "--no-root"];
+      let poetryInstallArgs = ["-m", "poetry"].concat(
+        getPoetryInstallArgs(reqOrSetupFile),
+      );
       if (isSecureMode) {
         poetryInstallArgs.unshift("-S");
       }
@@ -756,7 +882,7 @@ export async function getPipFrozenTree(
           thoughtLog(
             "Hmm, poetry doesn't seem to be available as a module. Perhaps it was installed directly 🤔?",
           );
-          poetryInstallArgs = ["install", "-n", "--no-root"];
+          poetryInstallArgs = getPoetryInstallArgs(reqOrSetupFile);
           // Attempt to perform poetry install
           result = safeSpawnSync("poetry", poetryInstallArgs, {
             cwd: basePath,