@cdxgen/cdxgen 13.5.0 → 13.6.0

package/lib/cli/jvmBom.js

@@ -1,7 +1,7 @@
 import { Buffer } from "node:buffer";
 import { readdirSync, readFileSync } from "node:fs";
 import { homedir } from "node:os";
-import { basename, dirname, join, resolve, sep } from "node:path";
+import { basename, dirname, join, relative, resolve, sep } from "node:path";
 import process from "node:process";
 
 import { build } from "@cdxgen/cdx-purl";
@@ -44,19 +44,24 @@ import {
   buildGradleCommandArguments,
   buildObjectForGradleModule,
   collectGradleDependencies,
+  excludeGradleBuildTooling,
   executeParallelGradleProperties,
   findMillDaemons,
   getGradleCommand,
   getMavenCommand,
   getMillCommand,
   getMillVersion,
+  gradleArgsFromEnv,
+  gradleDependencyTasks,
   hasGradleInvocationFailure,
   isProjectToolingAllowed,
+  loadGradleModulesForReports,
   parseGradleDep,
   parseGradleInfoLogsForUrls,
   parseGradleProperties,
   parseGradleResolvedDistributions,
   parseGradleVersionCatalog,
+  reconcileGradleComponents,
   recordGradleInvocationFailure,
   splitOutputByGradleProjects,
   stopMillDaemons,
@@ -91,6 +96,7 @@ import {
 } from "../ecosystems/sbtutils.js";
 import { collectScalaCliComponents } from "../ecosystems/scalacliutils.js";
 import { collectScalaJsNpmComponents } from "../ecosystems/scalasem.js";
+import { isInVendoredDir } from "../ecosystems/vendoredCode.js";
 import {
   isBuildToolRateLimited,
   noteBuildToolRateLimit,
@@ -99,6 +105,7 @@ import {
   collectJarNS,
   collectMvnDependencies,
   convertJarNSToPackages,
+  namespaceMappingOfComponents,
 } from "../inventory/deps.js";
 import { mergeDependencies, trimComponents } from "../inventory/depsUtils.js";
 import {
@@ -112,8 +119,20 @@ import {
   projectSettingsArgs,
 } from "../inventory/jvmLocalRepos.js";
 import { readGradleWrapperVersion } from "../inventory/jvmToolEnv.js";
-import { applyPurl, mavenPurl } from "../inventory/purl.js";
+import {
+  bazelMirrorArgs,
+  coursierMirrorEnv,
+  gradleMirrorInitScriptArgs,
+  mavenMirrorSettingsArgs,
+  mavenWrapperMirrorEnv,
+  sbtMirrorEnv,
+} from "../inventory/mavenCentralMirror.js";
+import { applyPurl, mavenPurl, tryBuildPurl } from "../inventory/purl.js";
 import { publishedArtifactId } from "../inventory/scalaCoords.js";
+import {
+  mergeScalaDuplicateComponents,
+  rekeyNamespaceMapping,
+} from "../inventory/scalaDedupe.js";
 import {
   buildBomNSData,
   createDefaultParentComponent,
@@ -253,6 +272,67 @@ export async function collectSbtDependencyJars(pkgList, known = {}) {
   return nsMapping;
 }
 
+/**
+ * Keep apart two independent Maven projects that declare the same
+ * coordinates, such as sample projects in `one/` and `two/` that both call
+ * themselves `com.example:demo:0.0.1-SNAPSHOT`. When `component` has the
+ * bom-ref of the parent or of a module already collected but comes from a
+ * different pom.xml, it gets the directory of its pom, relative to the scan
+ * root, as the purl subpath, and the dependency entries that name it follow.
+ *
+ * @param {Object} component Root component of a Maven tree, updated in place
+ * @param {Object} parentComponent Parent component and the modules under it
+ * @param {string} rootPath Directory being scanned
+ * @param {Object[]} dependenciesList Dependency entries of the tree, updated in place
+ * @returns {boolean} True when the component got a distinct identity
+ */
+export function separateDuplicateMavenProject(
+  component,
+  parentComponent,
+  rootPath,
+  dependenciesList = [],
+) {
+  const ref = component?.["bom-ref"];
+  if (!ref || !component.purl) {
+    return false;
+  }
+  const holder = [parentComponent, ...(parentComponent?.components || [])].find(
+    (candidate) => candidate?.["bom-ref"] === ref,
+  );
+  const srcFile = (c) =>
+    c?.properties?.find((p) => p.name === "internal:SrcFile")?.value;
+  const ownPom = srcFile(component);
+  const holderPom = srcFile(holder);
+  if (!ownPom || !holderPom || resolve(ownPom) === resolve(holderPom)) {
+    return false;
+  }
+  const subpath = relative(resolve(rootPath), dirname(resolve(ownPom)))
+    .split(sep)
+    .filter((segment) => segment && segment !== "..")
+    .join("/");
+  const purl =
+    subpath &&
+    tryBuildPurl({
+      type: "maven",
+      namespace: component.group || null,
+      name: component.name,
+      version: component.version || null,
+      qualifiers: component.qualifiers || { type: "jar" },
+      subpath,
+    });
+  if (!purl) {
+    return false;
+  }
+  applyPurl(component, purl);
+  const newRef = component["bom-ref"];
+  for (const entry of dependenciesList) {
+    if (entry.ref === ref) {
+      entry.ref = newRef;
+    }
+  }
+  return true;
+}
+
 /**
  * The purls and bom-refs of a parent component and the module components
  * nested under it.
@@ -814,6 +894,17 @@ export async function createJavaBom(path, options) {
         const projectSettings = projectSettingsArgs(basePath, {
           secureMode: isSecureModeActive(),
         });
+        // When the mirror switch is on, Maven resolves Central from the
+        // mirror through a generated global settings file; a project
+        // settings.xml is merged into that copy so its repositories and its
+        // own mirror survive. A settings file the user named keeps winning.
+        const mirrorSettingsArgs = mavenMirrorSettingsArgs(
+          projectSettings.settingsFile,
+        );
+        const mvnSettingsArgs = mirrorSettingsArgs.length
+          ? mirrorSettingsArgs
+          : projectSettings.args;
+        const mvnSpawnEnv = { ...process.env, ...mavenWrapperMirrorEnv() };
         if (projectSettings.args.length) {
           console.log(
             `Passing ${projectSettings.settingsFile} to Maven as global settings (-gs), so your own settings.xml still applies. Name a settings file in MVN_ARGS to change this.`,
@@ -839,9 +930,10 @@ export async function createJavaBom(path, options) {
         if (!useMavenDepsTree && !isBuildToolRateLimited("maven")) {
           thoughtLog("The user wants me to use the cyclonedx-maven plugin.");
           console.log(`Executing '${mavenCmd}' in`, basePath);
-          result = safeSpawnSync(mavenCmd, mvnArgs, {
+          result = safeSpawnSync(mavenCmd, mvnArgs.concat(mirrorSettingsArgs), {
             cwd: basePath,
             shell: isWin,
+            env: mvnSpawnEnv,
           });
           noteBuildToolRateLimit("maven", commandOutputText(result), {
             command: `${basename(mavenCmd)} ${mvnArgs.join(" ")}`,
@@ -906,7 +998,7 @@ export async function createJavaBom(path, options) {
               addArgs = parseMavenArgs(readEnvironmentVariable("MVN_ARGS"));
               mvnTreeArgs = mvnTreeArgs.concat(addArgs);
             }
-            mvnTreeArgs = mvnTreeArgs.concat(projectSettings.args);
+            mvnTreeArgs = mvnTreeArgs.concat(mvnSettingsArgs);
             // For the first pom alone, we need to execute first in non-recursive mode to capture
             // the parent component. Then, we execute all of them in recursive mode
             // A rate limit the --deep copy-dependencies run met ends it too.
@@ -922,11 +1014,12 @@ export async function createJavaBom(path, options) {
               ];
               findParentComponentArgs = findParentComponentArgs.concat(
                 addArgs,
-                projectSettings.args,
+                mvnSettingsArgs,
               );
               result = safeSpawnSync(mavenCmd, findParentComponentArgs, {
                 cwd: basePath,
                 shell: isWin,
+                env: mvnSpawnEnv,
               });
               noteBuildToolRateLimit("maven", commandOutputText(result), {
                 command: `${basename(mavenCmd)} ${findParentComponentArgs.join(" ")}`,
@@ -956,10 +1049,11 @@ export async function createJavaBom(path, options) {
                   "dependency:tree",
                   "-N",
                   `-DoutputFile=${tempMvnParentTreeText}`,
-                ].concat(addArgs, projectSettings.args);
+                ].concat(addArgs, mvnSettingsArgs);
                 result = safeSpawnSync(mavenCmd, findParentComponentArgs, {
                   cwd: basePath,
                   shell: isWin,
+                  env: mvnSpawnEnv,
                 });
               }
               if (result.status === 0) {
@@ -1004,6 +1098,7 @@ export async function createJavaBom(path, options) {
               result = safeSpawnSync(mavenCmd, mvnTreeArgs, {
                 cwd: basePath,
                 shell: isWin,
+                env: mvnSpawnEnv,
               });
               noteBuildToolRateLimit("maven", commandOutputText(result), {
                 command: `${basename(mavenCmd)} ${mvnTreeArgs.join(" ")}`,
@@ -1029,10 +1124,11 @@ export async function createJavaBom(path, options) {
                 mvnTreeArgs = [
                   "dependency:tree",
                   `-DoutputFile=${tempMvnTreeText}`,
-                ].concat(addArgs, projectSettings.args);
+                ].concat(addArgs, mvnSettingsArgs);
                 result = safeSpawnSync(mavenCmd, mvnTreeArgs, {
                   cwd: basePath,
                   shell: isWin,
+                  env: mvnSpawnEnv,
                 });
                 noteBuildToolRateLimit("maven", commandOutputText(result), {
                   command: `${basename(mavenCmd)} ${mvnTreeArgs.join(" ")}`,
@@ -1182,6 +1278,12 @@ export async function createJavaBom(path, options) {
                   parentComponent = tmpParentComponent;
                   parentComponent.components = [];
                 } else {
+                  separateDuplicateMavenProject(
+                    tmpParentComponent,
+                    parentComponent,
+                    path,
+                    parsedList.dependenciesList,
+                  );
                   parentComponent.components.push(tmpParentComponent);
                 }
                 if (parsedList?.dependenciesList?.length) {
@@ -1309,9 +1411,14 @@ export async function createJavaBom(path, options) {
       }
     }
     const allProjects = [];
-    const allProjectsAddedPurls = [];
+    // Modules keyed by bom-ref: a module without a purl still has a bom-ref
+    // of its own, while every such module shares an undefined purl.
+    const allProjectsAddedRefs = new Map();
     const rootDependsOn = new Set();
     const gradleModules = new Map();
+    // Included builds that build Gradle plugins, such as a build-logic build
+    // included from pluginManagement. Their classpath is build tooling.
+    const gradlePluginBuilds = new Set();
     // Determine the root path for gradle
     // Fixes gradle invocation for microservices-demo
     let gradleRootPath = path;
@@ -1413,6 +1520,12 @@ export async function createJavaBom(path, options) {
       const gradleInitArgs = readEnvironmentVariable("GRADLE_INCLUDED_BUILDS")
         ? []
         : ["--init-script", GRADLE_INIT_SCRIPT];
+      // The mirror init script joins cdxgen's own helper script; Gradle
+      // applies every init script it is given.
+      const gradleMirrorArgs = gradleMirrorInitScriptArgs();
+      if (gradleMirrorArgs.length) {
+        gradleInitArgs.push(...gradleMirrorArgs);
+      }
       if (resolveGradleDistribution && gradleInitArgs.length) {
         gradleInitArgs.push("-PcdxgenResolveDistribution=true");
       }
@@ -1484,12 +1597,19 @@ export async function createJavaBom(path, options) {
           );
           if (!includedBuilds.includes(key)) {
             parentComponent = rootComponent;
+          } else if (retMap.gradlePlugin) {
+            gradlePluginBuilds.add(key);
           }
           gradleModules.set(key, rootComponent);
-          if (!allProjectsAddedPurls.includes(rootComponent["purl"])) {
+          if (
+            !allProjectsAddedRefs.has(rootComponent["bom-ref"] ?? rootComponent)
+          ) {
             allProjects.push(rootComponent);
             rootDependsOn.add(rootComponent["bom-ref"]);
-            allProjectsAddedPurls.push(rootComponent["purl"]);
+            allProjectsAddedRefs.set(
+              rootComponent["bom-ref"] ?? rootComponent,
+              rootComponent,
+            );
           }
           allProjectsStr = allProjectsStr.concat(retMap.projects);
         }
@@ -1522,14 +1642,60 @@ export async function createJavaBom(path, options) {
           );
           const rootSubProject = retMap.rootProject;
           if (rootSubProject) {
+            // A subproject whose properties task printed nothing is named as
+            // Gradle publishes it by default: the last segment of its path,
+            // with the root project's group and version.
+            if (!splitPropTaskOut.has(subProject)) {
+              retMap.metadata.group ||= parentComponent?.group || "";
+              if (
+                retMap.metadata.version === "latest" &&
+                parentComponent?.version
+              ) {
+                retMap.metadata.version = parentComponent.version;
+              }
+            }
             const rootSubProjectObj = await buildObjectForGradleModule(
-              rootSubProject === "root" ? subProject : rootSubProject,
+              rootSubProject === "root"
+                ? subProject.split(":").pop()
+                : rootSubProject,
               retMap.metadata,
             );
-            if (!allProjectsAddedPurls.includes(rootSubProjectObj["purl"])) {
+            // Two subprojects that Gradle would publish under the same
+            // coordinates, such as `:a:util` and `:b:util` with a shared group
+            // and version, stay apart: the later one gets its project path as
+            // the purl subpath. A module that shares the root project's
+            // coordinates is the root's own artifact and merges with it.
+            const holder = allProjectsAddedRefs.get(
+              rootSubProjectObj["bom-ref"],
+            );
+            if (
+              holder &&
+              holder !== parentComponent &&
+              holder !== gradleModules.get(subProject)
+            ) {
+              const subpath = subProject.split(":").filter(Boolean).join("/");
+              const distinctPurl = rootSubProjectObj.purl
+                ? tryBuildPurl({
+                    type: "maven",
+                    namespace: rootSubProjectObj.group || null,
+                    name: rootSubProjectObj.name,
+                    version: rootSubProjectObj.version || null,
+                    qualifiers: { type: "jar" },
+                    subpath,
+                  })
+                : null;
+              if (distinctPurl) {
+                applyPurl(rootSubProjectObj, distinctPurl);
+              } else {
+                rootSubProjectObj["bom-ref"] =
+                  `${rootSubProjectObj["bom-ref"]}#${subpath}`;
+              }
+            }
+            const moduleKey = rootSubProjectObj["bom-ref"] ?? rootSubProjectObj;
+            if (!allProjectsAddedRefs.has(moduleKey)) {
               allProjects.push(rootSubProjectObj);
               rootDependsOn.add(rootSubProjectObj["bom-ref"]);
-              allProjectsAddedPurls.push(rootSubProjectObj["purl"]);
+              allProjectsAddedRefs.set(moduleKey, rootSubProjectObj);
             }
             gradleModules.set(subProject, rootSubProjectObj);
           }
@@ -1561,7 +1727,6 @@ export async function createJavaBom(path, options) {
         ? readEnvironmentVariable("GRADLE_DEPENDENCY_TASK")
         : "dependencies";
 
-      const gradleSubCommands = [];
       let modulesToSkip = readEnvironmentVariable("GRADLE_SKIP_MODULES")
         ? readEnvironmentVariable("GRADLE_SKIP_MODULES").split(",")
         : [];
@@ -1570,23 +1735,16 @@ export async function createJavaBom(path, options) {
           readEnvironmentVariable("GRADLE_SKIP_MODULE_DEPENDENCIES").split(","),
         );
       }
-      if (!modulesToSkip.includes("root")) {
-        gradleSubCommands.push(gradleDepTask);
-      }
-      for (const [key, sp] of gradleModules) {
-        //create single command for dependencies tasks on all subprojects
-        if (sp.purl !== parentComponent.purl && !modulesToSkip.includes(key)) {
-          gradleSubCommands.push(`${key}:${gradleDepTask}`);
-        }
-      }
+      const gradleSubCommands = gradleDependencyTasks(
+        gradleModules,
+        parentComponent,
+        gradleDepTask,
+        modulesToSkip,
+      );
       const gradleArguments = buildGradleCommandArguments(
-        readEnvironmentVariable("GRADLE_ARGS")
-          ? readEnvironmentVariable("GRADLE_ARGS").split(" ")
-          : [],
+        gradleArgsFromEnv("GRADLE_ARGS"),
         gradleSubCommands,
-        readEnvironmentVariable("GRADLE_ARGS_DEPENDENCIES")
-          ? readEnvironmentVariable("GRADLE_ARGS_DEPENDENCIES").split(" ")
-          : [],
+        gradleArgsFromEnv("GRADLE_ARGS_DEPENDENCIES"),
         gradleCmd.length,
       );
       const allOutputs = [];
@@ -1647,14 +1805,30 @@ export async function createJavaBom(path, options) {
         const perProjectOutput = splitOutputByGradleProjects(cmdOutput, [
           gradleDepTask,
         ]);
+        await loadGradleModulesForReports(
+          perProjectOutput.values(),
+          gradleModules,
+          gradleRootPath,
+          options,
+        );
+        const reportedProjects = new Set(perProjectOutput.keys());
         for (const key of gradleModules.keys()) {
           const parsedList = await parseGradleDep(
             perProjectOutput.has(key) ? perProjectOutput.get(key) : "",
             key,
             gradleModules,
             gradleRootPath,
+            reportedProjects,
           );
           const dlist = parsedList.pkgList;
+          if (
+            dlist?.length &&
+            [...gradlePluginBuilds].some(
+              (build) => key === build || key.startsWith(`${build}:`),
+            )
+          ) {
+            excludeGradleBuildTooling(dlist, gradleModules);
+          }
           if (parsedList?.dependenciesList?.length) {
             dependencies = mergeDependencies(
               dependencies,
@@ -1672,6 +1846,7 @@ export async function createJavaBom(path, options) {
             pkgList = pkgList.concat(dlist);
           }
         }
+        reconcileGradleComponents(pkgList);
       }
       if (pkgList.length) {
         const fileToUrlMap = parseGradleInfoLogsForUrls(sstdout);
@@ -1685,9 +1860,15 @@ export async function createJavaBom(path, options) {
           if (!p.externalReferences) {
             p.externalReferences = [];
           }
-          if (!p.externalReferences.find((r) => r.type === "distribution")) {
+          // The repository URLs name jar files, which a platform does not
+          // have. The leading slash keeps group `foo` from matching a URL
+          // under `org/foo/`.
+          if (
+            p.qualifiers?.type !== "pom" &&
+            !p.externalReferences.find((r) => r.type === "distribution")
+          ) {
             const groupPath = p.group.replace(/\./g, "/");
-            const matchPath = `${groupPath}/${p.name}/${p.version}/`;
+            const matchPath = `/${groupPath}/${p.name}/${p.version}/`;
             let matchedUrl = Object.values(fileToUrlMap).find((url) =>
               url.includes(matchPath),
             );
@@ -1809,11 +1990,14 @@ export async function createJavaBom(path, options) {
     // Bazel
     // Look for the BUILD file only in the root directory
     // NOTE: This can match BUILD files used by perl, so could lead to errors in some projects
+    // A workspace under third_party/ or vendor/ is a copied project's: its
+    // bazel_dep entries are that project's build tooling, and building it
+    // would run its build.
     const bazelFiles = getAllFiles(
       path,
       `${options.multiProject ? "**/" : ""}{WORKSPACE{,.bazel},MODULE.bazel}`,
       options,
-    );
+    ).filter((f) => !isInVendoredDir(path, f));
     if (
       bazelFiles?.length &&
       !hasAnyProjectType(
@@ -1886,6 +2070,9 @@ export async function createJavaBom(path, options) {
         ) {
           bArgs = ["--bazelrc=.bazelrc", "build", bazelTarget];
         }
+        // Repository rules see the mirror through their own environment,
+        // which only a --repo_env flag reaches.
+        bArgs = bArgs.concat(bazelMirrorArgs());
         console.log("Executing", BAZEL_CMD, "in", basePath);
         let result = safeSpawnSync(BAZEL_CMD, bArgs, {
           cwd: basePath,
@@ -2099,7 +2286,7 @@ export async function createJavaBom(path, options) {
         // sbt resolves into the user's own caches, so a project that has been
         // built resolves without the network. The dependency trees download
         // the jar of every dependency they list, which is all --deep reads.
-        const env = { ...process.env };
+        const env = { ...process.env, ...sbtMirrorEnv() };
         for (const i in sbtProjects) {
           if (isBuildToolRateLimited("sbt")) {
             // Every remaining sbt build in this scan resolves from the same
@@ -2450,19 +2637,13 @@ export async function createJavaBom(path, options) {
         const jarNSData = await createJarBom(path, options);
         if (jarNSData?.bomJson?.components) {
           pkgList = pkgList.concat(jarNSData?.bomJson?.components);
-          const targetJarNSMapping = {};
-          for (const p of jarNSData.bomJson.components) {
-            if (!p?.purl || !p?.properties?.length) {
-              continue;
-            }
-            const nsProp = p.properties.filter(
-              (prop) => prop.name === "internal:Namespaces",
-            );
-            if (nsProp.length) {
-              targetJarNSMapping[p.purl] = nsProp[0].value;
-            }
-          }
-          jarNSMapping = { ...jarNSMapping, ...targetJarNSMapping };
+          jarNSMapping = {
+            ...jarNSMapping,
+            ...namespaceMappingOfComponents(
+              jarNSData.bomJson.components,
+              jarNSMapping,
+            ),
+          };
         }
       }
     }
@@ -2504,10 +2685,12 @@ export async function createJavaBom(path, options) {
       if (DEBUG_MODE) {
         console.log("Executing", millCmd, "in", millRootPath);
       }
+      const millSpawnEnv = { ...process.env, ...coursierMirrorEnv() };
       let sresult = safeSpawnSync(millCmd, millArgs, {
         cwd: millRootPath,
         shell: isWin,
         maxBuffer: MAX_BUFFER * 10,
+        env: millSpawnEnv,
       });
       noteBuildToolRateLimit("mill", commandOutputText(sresult), {
         command: `${basename(millCmd)} ${millArgs.join(" ")}`,
@@ -2546,6 +2729,7 @@ export async function createJavaBom(path, options) {
       sresult = safeSpawnSync(millCmd, millResolveArgs, {
         cwd: millRootPath,
         shell: isWin,
+        env: millSpawnEnv,
       });
       if (sresult.status !== 0 || sresult.error) {
         if (options.failOnError || DEBUG_MODE) {
@@ -2710,6 +2894,13 @@ export async function createJavaBom(path, options) {
   // Deduplication and metadata belong to whatever was collected, so they run
   // on the deferred path too.
   pkgList = trimComponents(pkgList);
+  // A library the build tool reported and a jar on disk named again is one
+  // component, wherever the jar scan spelled it differently. The jar
+  // namespace map follows the purls the merge settled on.
+  const scalaMerge = mergeScalaDuplicateComponents(pkgList, dependencies);
+  pkgList = scalaMerge.components;
+  dependencies = scalaMerge.dependencies;
+  jarNSMapping = rekeyNamespaceMapping(jarNSMapping, scalaMerge.purlMap);
   // --deep collects namespaces; it does not opt in to remote licence lookups,
   // which FETCH_LICENSE controls. The project's own modules are never looked
   // up remotely: they are not published.