@cdxgen/cdxgen 13.3.0 → 13.5.0

package/lib/ecosystems/parsers-jvm.js

@@ -1,27 +1,80 @@
-import { readFileSync } from "node:fs";
-import { basename, dirname, resolve } from "node:path";
+import { readdirSync, readFileSync } from "node:fs";
+import {
+  basename,
+  dirname,
+  isAbsolute,
+  join,
+  relative,
+  resolve,
+} from "node:path";
 
 import { build } from "@cdxgen/cdx-purl";
 import { parseEDNString } from "edn-data";
 
-import { readEnvironmentVariable } from "../core/activity.js";
+import { DEBUG_MODE, readEnvironmentVariable } from "../core/activity.js";
 import { recordDegradation } from "../core/buildLedger.js";
 import { includeMavenTestScope } from "../core/env.js";
+import { safeExistsSync } from "../core/fs.js";
 import { thoughtLog } from "../core/logger.js";
 import { applyPurl } from "../inventory/purl.js";
+import { parseScalaArtifact } from "../inventory/scalaCoords.js";
 import { xml2js } from "../parsers/xml.js";
 
+/** Bound on the `${...}` references one pom value may expand through. */
+const POM_PROPERTY_EXPANSION_LIMIT = 10;
+
+/**
+ * Expand the `${name}` references in a pom value. The value is returned only
+ * when every reference resolves, so a version is never half expanded.
+ *
+ * @param {string} value Raw pom value
+ * @param {Function} lookup Returns the value of a property name, or undefined
+ * @returns {string|undefined} Expanded value, or undefined
+ */
+function expandPomValue(value, lookup) {
+  let expanded = value;
+  for (
+    let i = 0;
+    i < POM_PROPERTY_EXPANSION_LIMIT && expanded?.includes("${");
+    i++
+  ) {
+    const start = expanded.indexOf("${");
+    const end = expanded.indexOf("}", start + 2);
+    if (end < 0) {
+      return undefined;
+    }
+    const replacement = lookup(expanded.slice(start + 2, end));
+    if (typeof replacement !== "string") {
+      return undefined;
+    }
+    expanded = `${expanded.slice(0, start)}${replacement}${expanded.slice(end + 1)}`;
+  }
+  return expanded?.includes("${") ? undefined : expanded;
+}
+
 /**
  * Parse pom file
  *
+ * Versions resolve from the pom's own properties and dependencyManagement.
+ * A caller that has read the pom's parents passes what they define in
+ * `inherited`, so versions the parents manage resolve too. Imported BOMs and
+ * parents outside the scanned directory are not read.
+ *
  * @param {string} pomFile pom file to parse
- * @returns {Object} Object containing pom properties, modules, and array of dependencies
+ * @param {Object} [inherited] Properties and managed versions from parent poms
+ * @param {Object} [inherited.properties] Property name to raw value
+ * @param {Object} [inherited.managedVersions] `groupId:artifactId` to version
+ * @returns {Object} Object containing pom properties, modules, the parent
+ *   reference, the user properties and managed versions this pom defines, and
+ *   array of dependencies
  */
-export function parsePom(pomFile) {
+export function parsePom(pomFile, inherited = {}) {
   const deps = [];
   let modules;
   let pomPurl;
   const properties = {};
+  const userProperties = {};
+  const managedVersions = {};
   let isQuarkus = false;
   const xmlData = readFileSync(pomFile, "utf-8");
   const project = xml2js(xmlData, {
@@ -54,25 +107,30 @@ export function parsePom(pomFile) {
     }
   }
   // Take the version from the parent if available
-  if (!properties.version && project.parent) {
+  if (!properties.version && project?.parent?.version?._) {
     properties.version = project.parent.version._;
   }
   // Take the groupId from the parent if available
-  if (!properties.groupId && project.parent) {
+  if (!properties.groupId && project?.parent?.groupId?._) {
     properties.groupId = project.parent.groupId._;
   }
+  let parent;
+  if (project?.parent) {
+    parent = {
+      groupId: project.parent.groupId?._,
+      artifactId: project.parent.artifactId?._,
+      version: project.parent.version?._,
+      // An empty <relativePath/> tells Maven not to look for the parent on
+      // disk; a missing one means ../pom.xml.
+      relativePath:
+        project.parent.relativePath === undefined
+          ? "../pom.xml"
+          : project.parent.relativePath?._ || "",
+    };
+  }
   if (project?.scm?.url?._) {
     properties.scm = project.scm.url._;
   }
-  if (properties.groupId || properties.artifactId) {
-    pomPurl = build({
-      type: "maven",
-      namespace: properties.groupId || "" || null,
-      name: properties.artifactId,
-      version: properties.version || null,
-      qualifiers: { type: properties.packaging || "jar" } || null,
-    });
-  }
   const moduleEntries = [];
   if (project?.modules?.module) {
     moduleEntries.push(project.modules.module);
@@ -91,11 +149,80 @@ export function parsePom(pomFile) {
   if (project?.properties) {
     for (const aprop of Object.keys(project.properties)) {
       properties[aprop] = project.properties[aprop]?._;
+      userProperties[aprop] = project.properties[aprop]?._;
       if (!isQuarkus && aprop.startsWith("quarkus.platform")) {
         isQuarkus = true;
       }
     }
   }
+  const inheritedProperties = inherited?.properties || {};
+  const lookupProperty = (name) => {
+    switch (name) {
+      case "project.version":
+      case "pom.version":
+      case "version":
+        return properties.version;
+      case "project.groupId":
+      case "pom.groupId":
+      case "groupId":
+        return properties.groupId;
+      case "project.artifactId":
+      case "pom.artifactId":
+      case "artifactId":
+        return properties.artifactId;
+      case "project.parent.version":
+        return parent?.version;
+      case "project.parent.groupId":
+        return parent?.groupId;
+      default:
+        return Object.hasOwn(userProperties, name)
+          ? userProperties[name]
+          : inheritedProperties[name];
+    }
+  };
+  // A property can refer to another, so each lookup expands its value too.
+  const expand = (value) =>
+    expandPomValue(value, (name) => {
+      const raw = lookupProperty(name);
+      return raw?.includes("${")
+        ? expandPomValue(raw, (inner) =>
+            inner === name ? undefined : lookupProperty(inner),
+          )
+        : raw;
+    });
+  let managedEntries = project?.dependencyManagement?.dependencies?.dependency;
+  if (managedEntries && !Array.isArray(managedEntries)) {
+    managedEntries = [managedEntries];
+  }
+  for (const amanaged of managedEntries || []) {
+    // An imported BOM manages versions this pom cannot see without fetching it
+    if (amanaged?.scope?._ === "import" || !amanaged?.version?._) {
+      continue;
+    }
+    const group = expand(amanaged.groupId?._);
+    const name = expand(amanaged.artifactId?._);
+    const version = expand(amanaged.version._);
+    if (group && name && version) {
+      managedVersions[`${group}:${name}`] = version;
+    }
+  }
+  const inheritedManagedVersions = inherited?.managedVersions || {};
+  // CI friendly versions such as ${revision} resolve from the properties
+  for (const coordinate of ["groupId", "version"]) {
+    if (properties[coordinate]?.includes("${")) {
+      properties[coordinate] =
+        expand(properties[coordinate]) || properties[coordinate];
+    }
+  }
+  if (properties.groupId || properties.artifactId) {
+    pomPurl = build({
+      type: "maven",
+      namespace: properties.groupId || "" || null,
+      name: properties.artifactId,
+      version: properties.version || null,
+      qualifiers: { type: properties.packaging || "jar" } || null,
+    });
+  }
   // Check the plugins for quarkus
   if (!isQuarkus && project?.build?.plugins?.plugin) {
     if (Array.isArray(project.build.plugins.plugin)) {
@@ -129,7 +256,18 @@ export function parsePom(pomFile) {
         versionStr = version._;
       }
       if (versionStr?.includes("$")) {
-        versionStr = properties[versionStr?.replace(/[${}]/g, "")];
+        versionStr = expand(versionStr);
+      }
+      let groupStr = adep.groupId ? adep.groupId._ : "";
+      if (groupStr?.includes("${")) {
+        groupStr = expand(groupStr) || groupStr;
+      }
+      const nameStr = adep.artifactId ? adep.artifactId._ : "";
+      if (!versionStr && !version?._) {
+        const managedKey = `${groupStr}:${nameStr}`;
+        versionStr = Object.hasOwn(managedVersions, managedKey)
+          ? managedVersions[managedKey]
+          : inheritedManagedVersions[managedKey];
       }
       const scope = adep.scope?._;
       const type = adep.type?._ || "jar";
@@ -147,8 +285,8 @@ export function parsePom(pomFile) {
       }
       if (includeMavenTestScope || scope !== "test") {
         deps.push({
-          group: adep.groupId ? adep.groupId._ : "",
-          name: adep.artifactId ? adep.artifactId._ : "",
+          group: groupStr,
+          name: nameStr,
           version: versionStr,
           qualifiers,
           scope: mapMavenScope(scope, optional === "true"),
@@ -170,7 +308,272 @@ export function parsePom(pomFile) {
       }
     }
   }
-  return { isQuarkus, pomPurl, modules, properties, dependencies: deps };
+  return {
+    isQuarkus,
+    pomPurl,
+    modules,
+    parent,
+    properties,
+    userProperties,
+    managedVersions,
+    dependencies: deps,
+  };
+}
+
+/** Bound on the chain of parent poms read from disk. */
+const LOCAL_PARENT_DEPTH_LIMIT = 10;
+
+/**
+ * The pom file a `<parent>` reference points at on disk, when it lies inside
+ * the scanned directory and declares the coordinates the reference names.
+ *
+ * @param {string} pomFile Child pom file
+ * @param {Object} parentRef Parent reference from parsePom
+ * @param {string} rootDir Scanned directory
+ * @returns {string|undefined} Parent pom file
+ */
+function locateLocalParentPom(pomFile, parentRef, rootDir) {
+  if (!parentRef?.relativePath || !rootDir) {
+    return undefined;
+  }
+  let candidate = resolve(dirname(pomFile), parentRef.relativePath);
+  if (!candidate.endsWith(".xml")) {
+    candidate = join(candidate, "pom.xml");
+  }
+  const rel = relative(resolve(rootDir), candidate);
+  if (!rel || rel.startsWith("..") || isAbsolute(rel)) {
+    return undefined;
+  }
+  if (!safeExistsSync(candidate)) {
+    return undefined;
+  }
+  try {
+    const { groupId, artifactId } = parsePom(candidate).properties || {};
+    if (
+      artifactId !== parentRef.artifactId ||
+      (parentRef.groupId && groupId !== parentRef.groupId)
+    ) {
+      return undefined;
+    }
+  } catch (_err) {
+    return undefined;
+  }
+  return candidate;
+}
+
+/**
+ * What a pom passes on to its children: its user properties and managed
+ * versions over those of its own local parents.
+ *
+ * @param {string} pomFile Pom file
+ * @param {string} rootDir Scanned directory
+ * @param {Map} cache Results by pom file
+ * @param {number} depth Current depth of the parent chain
+ * @returns {Object} Inherited properties and managed versions
+ */
+function localPomInheritance(pomFile, rootDir, cache, depth) {
+  if (cache.has(pomFile)) {
+    return cache.get(pomFile);
+  }
+  const empty = { properties: {}, managedVersions: {} };
+  // A parent chain that loops back reads as having no parent
+  cache.set(pomFile, empty);
+  let inherited = empty;
+  try {
+    const parentPom =
+      depth < LOCAL_PARENT_DEPTH_LIMIT
+        ? locateLocalParentPom(pomFile, parsePom(pomFile).parent, rootDir)
+        : undefined;
+    if (parentPom) {
+      inherited = localPomInheritance(parentPom, rootDir, cache, depth + 1);
+    }
+    const own = parsePom(pomFile, inherited);
+    const result = {
+      properties: { ...inherited.properties, ...own.userProperties },
+      managedVersions: { ...inherited.managedVersions, ...own.managedVersions },
+    };
+    cache.set(pomFile, result);
+    return result;
+  } catch (_err) {
+    return empty;
+  }
+}
+
+/**
+ * Parse a pom file together with the parent poms it names that are present
+ * inside the scanned directory, without running Maven. Properties and
+ * dependencyManagement versions from those parents resolve the versions the
+ * pom leaves to them. Parents outside the directory and imported BOMs are not
+ * read, so the versions they manage stay empty.
+ *
+ * @param {string} pomFile Pom file to parse
+ * @param {string} rootDir Scanned directory; parents outside it are ignored
+ * @param {Map} [cache] Reused across the poms of one scan
+ * @returns {Object} Same shape as parsePom
+ */
+export function parsePomWithLocalParents(pomFile, rootDir, cache = new Map()) {
+  const pomPath = resolve(pomFile);
+  const parentPom = locateLocalParentPom(
+    pomPath,
+    parsePom(pomPath).parent,
+    rootDir,
+  );
+  if (!parentPom) {
+    return parsePom(pomFile);
+  }
+  return parsePom(pomFile, localPomInheritance(parentPom, rootDir, cache, 1));
+}
+
+/** Bound on the reactor walk. A pom's module list is untrusted input, so the
+ * walk cannot trust it to stay finite. */
+const REACTOR_MODULE_LIMIT = 500;
+
+/**
+ * Every pom file of the reactor an aggregator pom builds: its own modules,
+ * recursively through nested aggregators. A recursive dependency:tree run
+ * from the aggregator resolves all of them, so their own runs repeat it.
+ *
+ * A module entry names a directory relative to its pom, sometimes a direct
+ * path to a pom file. Entries that are absolute, or that climb out of the
+ * aggregator's directory, are ignored: the scan never listed them, and the
+ * walk must not follow a pom's pointers outside its own tree.
+ *
+ * @param {string} rootPom Path of the aggregator pom.
+ * @returns {Set<string>} Pom files of the reactor, excluding the aggregator.
+ */
+export function resolveReactorPomFiles(rootPom) {
+  const pomFiles = new Set();
+  const visited = new Set();
+  const rootDir = dirname(resolve(rootPom));
+  const queue = [resolve(rootPom)];
+  while (queue.length > 0 && pomFiles.size < REACTOR_MODULE_LIMIT) {
+    const pom = queue.shift();
+    if (visited.has(pom)) {
+      continue;
+    }
+    visited.add(pom);
+    let modules;
+    try {
+      modules = parsePom(pom)?.modules;
+    } catch (_err) {
+      // An unreadable module pom ends this branch of the walk.
+      continue;
+    }
+    for (const module of modules || []) {
+      if (typeof module !== "string") {
+        continue;
+      }
+      const cleaned = module.trim().replaceAll("\\", "/");
+      if (
+        !cleaned ||
+        cleaned.includes("\0") ||
+        cleaned.startsWith("/") ||
+        /^[A-Za-z]:/.test(cleaned)
+      ) {
+        continue;
+      }
+      const modulePom = resolve(
+        dirname(pom),
+        cleaned.endsWith(".xml") ? cleaned : join(cleaned, "pom.xml"),
+      );
+      const withinRoot = relative(rootDir, modulePom);
+      if (
+        !withinRoot ||
+        withinRoot.startsWith("..") ||
+        isAbsolute(withinRoot) ||
+        !safeExistsSync(modulePom)
+      ) {
+        continue;
+      }
+      if (visited.has(modulePom)) {
+        continue;
+      }
+      pomFiles.add(modulePom);
+      queue.push(modulePom);
+    }
+  }
+  return pomFiles;
+}
+
+/**
+ * File name stem for the trees of one recursive dependency:tree run. Maven
+ * expands the project expressions for each reactor module, so every module
+ * writes its own file; a fixed name would hold only the last module's tree.
+ */
+export const MAVEN_MODULE_TREE_STEM =
+  // biome-ignore lint/suspicious/noTemplateCurlyInString: Maven expands these project expressions itself
+  "${project.groupId}-${project.artifactId}";
+
+/**
+ * The groupId-artifactId key a module's tree file is named by.
+ *
+ * @param {string} pomFile Path of the module pom.
+ * @returns {string|undefined} The key, or undefined when the pom cannot be read.
+ */
+function moduleTreeKey(pomFile) {
+  try {
+    const { groupId, artifactId } = parsePom(pomFile)?.properties || {};
+    return groupId && artifactId ? `${groupId}-${artifactId}` : undefined;
+  } catch (_err) {
+    return undefined;
+  }
+}
+
+/**
+ * Read the trees one dependency:tree run wrote, one file per reactor module
+ * (see MAVEN_MODULE_TREE_STEM), and tell which of the reactor's pom files
+ * they cover. The pom the run started from comes first, then its modules in
+ * reactor order. A module pom counts as covered only when its own tree is
+ * among the files, so a module the run left out still runs on its own.
+ *
+ * @param {string} treeDir Directory the run wrote its trees into.
+ * @param {string} rootPom The pom file the run started from.
+ * @returns {{trees: Array<{pomFile: string, text: string}>, covered: Set<string>}}
+ */
+export function readMavenModuleTrees(treeDir, rootPom) {
+  const trees = [];
+  const covered = new Set();
+  let names;
+  try {
+    names = readdirSync(treeDir).sort();
+  } catch (_err) {
+    return { trees, covered };
+  }
+  const byKey = new Map();
+  for (const name of names) {
+    const stem = name.replace(/\.(json|txt)$/, "");
+    if (stem !== name && !byKey.has(stem)) {
+      byKey.set(stem, name);
+    }
+  }
+  const take = (key, pomFile) => {
+    const name = byKey.get(key);
+    if (!name) {
+      return false;
+    }
+    byKey.delete(key);
+    trees.push({
+      pomFile,
+      text: readFileSync(join(treeDir, name), { encoding: "utf-8" }),
+    });
+    return true;
+  };
+  const rootKey = moduleTreeKey(rootPom);
+  if (rootKey) {
+    take(rootKey, rootPom);
+  }
+  for (const modulePom of resolveReactorPomFiles(rootPom)) {
+    const key = moduleTreeKey(modulePom);
+    if (key && take(key, modulePom)) {
+      covered.add(modulePom);
+    }
+  }
+  // Trees no pom of the reactor names, such as the single file of a Maven
+  // that did not expand the expressions, are kept under the root pom.
+  for (const key of [...byKey.keys()]) {
+    take(key, rootPom);
+  }
+  return { trees, covered };
 }
 
 function mapMavenScope(componentScope, isOptional = false) {
@@ -471,10 +874,16 @@ export function parseMavenTree(rawOutput, pomFile) {
 /**
  * Parse mill dependencies from file
  *
+ * Scala artifacts are emitted in the sbt purl form: the Scala binary suffix
+ * is stripped from the artifact name and the version it carried is recorded
+ * in the cdx:scala:compilerVersion property, so one library keeps a single
+ * purl whichever build tool reported it.
+ *
  * @param {string} module name of the module
  * @param {map} dependencies the parsed dependencies
  * @param {map} relations a map containing all relations
  * @param {string} millRootPath root of the project
+ * @param {string} logFileName name of the tree log the module wrote
  *
  * @returns the bom-ref of the module
  */
@@ -483,6 +892,7 @@ export function parseMillDependency(
   dependencies,
   relations,
   millRootPath,
+  logFileName = "ivyDepsTree.log",
 ) {
   const treeRegex = /^(?<treeIndentation>(?:[?├│└─ ]{3})+)(?<dependency>.*)/m;
   const ESC = "\\x1B";
@@ -522,12 +932,7 @@ export function parseMillDependency(
     // Now put the versions we removed above back into the path and replace the brackets at the same time
     moduleFilePath = moduleFilePath.replace("[]", `/${versionNumber}/`);
   }
-  moduleFilePath = resolve(
-    millRootPath,
-    "out",
-    moduleFilePath,
-    "ivyDepsTree.log",
-  );
+  moduleFilePath = resolve(millRootPath, "out", moduleFilePath, logFileName);
   moduleComponent.properties = [
     {
       name: "internal:SrcFile",
@@ -547,7 +952,27 @@ export function parseMillDependency(
       ],
     },
   };
-  const dependencyTreeLog = readFileSync(moduleFilePath, "utf-8");
+  let dependencyTreeLog;
+  try {
+    dependencyTreeLog = readFileSync(moduleFilePath, "utf-8");
+  } catch (_err) {
+    // A cross-published module writes its log under the dotted version
+    // directory (0.23.38) while the resolve output prints it with
+    // underscores (0_23_38)
+    try {
+      dependencyTreeLog = readFileSync(
+        moduleFilePath.replace(/(\d+)_(\d+)_(\d+)/, "$1.$2.$3"),
+        "utf-8",
+      );
+    } catch (_err2) {
+      if (DEBUG_MODE) {
+        console.log(
+          `No Mill dependency tree log was written for the module ${module}.`,
+        );
+      }
+      return moduleComponent["bom-ref"];
+    }
+  }
   const dependencyTreeLines = dependencyTreeLog
     .trim()
     .split("\n")
@@ -568,11 +993,20 @@ export function parseMillDependency(
       [group, name, version] = match.groups.dependency.split(":");
       version = versionRegex.exec(version).groups.version;
     }
+    const scalaInfo = parseScalaArtifact(name);
     const component = completeComponent({
       group,
-      name,
+      name: scalaInfo.purlName,
       version,
     });
+    if (scalaInfo.binaryVersion) {
+      component.properties = [
+        {
+          name: "cdx:scala:compilerVersion",
+          value: scalaInfo.binaryVersion,
+        },
+      ];
+    }
     if (!dependencies.has(component["bom-ref"])) {
       dependencies.set(component["bom-ref"], component);
       relations.set(component["bom-ref"], []);
@@ -587,7 +1021,7 @@ export function parseMillDependency(
   return moduleComponent["bom-ref"];
 }
 
-function completeComponent(component) {
+export function completeComponent(component) {
   component["type"] = component.group ? "library" : "application";
   try {
     const purl = build({