@cdxgen/cdxgen 13.3.0 → 13.4.0

package/lib/ecosystems/sbtutils.js

@@ -1,11 +1,19 @@
 import { constants, readdirSync, readFileSync } from "node:fs";
-import { join } from "node:path";
+import { delimiter, join } from "node:path";
 import process from "node:process";
 
 import { build } from "@cdxgen/cdx-purl";
-import { valid } from "semver";
+import { major, valid } from "semver";
 
+import { isWin } from "../core/paths.js";
+import { registerRunReset } from "../core/runState.js";
+import {
+  coursierCacheDir,
+  locateInCoursierCache,
+  resetJvmLocalRepoCaches,
+} from "../inventory/jvmLocalRepos.js";
 import { applyPurl, mavenPurl } from "../inventory/purl.js";
+import { parseScalaArtifact } from "../inventory/scalaCoords.js";
 
 /**
  * A lightweight parser for Java .properties files, replacing the properties-reader library.
@@ -61,6 +69,7 @@ import {
   multiChecksumFile,
   safeCopyFileSync,
   safeExistsSync,
+  safeSpawnSync,
   safeUnlinkSync,
   safeWriteSync,
 } from "../core/fs.js";
@@ -152,174 +161,6 @@ export function cleanupPlugin(projectPath, originalPluginsFile) {
   return false;
 }
 
-/**
- * Returns the root directory of the local Coursier cache, or null when it
- * cannot be determined or does not exist.
- *
- * @returns {string|null} Absolute path to the cache root
- */
-function coursierCacheRoot() {
-  const configured = readEnvironmentVariable("COURSIER_CACHE");
-  let cacheRoot = configured;
-  if (!cacheRoot) {
-    const home =
-      readEnvironmentVariable("HOME") || readEnvironmentVariable("USERPROFILE");
-    if (!home) {
-      return null;
-    }
-    if (process.platform === "darwin") {
-      cacheRoot = join(home, "Library", "Caches", "Coursier", "v1");
-    } else if (process.platform === "win32") {
-      const localAppData = readEnvironmentVariable("LOCALAPPDATA");
-      if (localAppData) {
-        cacheRoot = join(localAppData, "Coursier", "Cache", "v1");
-      } else {
-        cacheRoot = join(home, "AppData", "Local", "Coursier", "Cache", "v1");
-      }
-    } else {
-      cacheRoot = join(home, ".cache", "coursier", "v1");
-    }
-  }
-  return safeExistsSync(cacheRoot) ? cacheRoot : null;
-}
-
-// Coursier stores an artifact at
-// <cacheRoot>/<protocol>/<host>/<repo path...>/<groupPath>/<name>/<version>/,
-// so the only unknown part of the path is the repository prefix. Those prefixes
-// are enumerated once per cache root with a shallow, depth-bounded scan and then
-// probed directly. The previous implementation instead ran a recursive `**` glob
-// across the whole cache for every single coordinate - twice, once for the
-// registry URL and once for the jar - which costs O(cache size) per lookup
-// regardless of whether the artifact is present. On a large shared CI cache that
-// turned a few hundred packages into tens of minutes of directory walking.
-// See https://github.com/cdxgen/cdxgen/issues/4291
-const _repoRootsCache = new Map();
-
-// A repository prefix always starts with <protocol>/<host>, and in practice adds
-// at most a couple of path segments ("maven2", "content/repositories/releases").
-// The scan stops at this depth so that it never descends into the group-id
-// directories, which is where the bulk of a cache lives.
-const MAX_REPO_PREFIX_DEPTH = 5;
-
-function coursierRepoRoots(cacheRoot) {
-  if (_repoRootsCache.has(cacheRoot)) {
-    return _repoRootsCache.get(cacheRoot);
-  }
-  const roots = [];
-  // Breadth-first so that shallower prefixes, which are the more common ones,
-  // are probed first.
-  let frontier = [[]];
-  for (let depth = 1; depth <= MAX_REPO_PREFIX_DEPTH; depth++) {
-    const next = [];
-    for (const parts of frontier) {
-      let entries;
-      try {
-        entries = readdirSync(join(cacheRoot, ...parts), {
-          withFileTypes: true,
-        });
-      } catch (_err) {
-        continue;
-      }
-      for (const entry of entries) {
-        if (!entry.isDirectory()) {
-          continue;
-        }
-        const childParts = [...parts, entry.name];
-        // <protocol>/<host> is the shortest thing that can be a repository.
-        if (depth >= 2) {
-          roots.push(childParts);
-        }
-        next.push(childParts);
-      }
-    }
-    frontier = next;
-    if (!frontier.length) {
-      break;
-    }
-  }
-  _repoRootsCache.set(cacheRoot, roots);
-  return roots;
-}
-
-/**
- * Converts a repository prefix in the Coursier cache layout into a URL.
- *
- * @param {string[]} parts Path segments of the prefix, relative to the cache root
- * @returns {string|null} The repository URL
- */
-function repoPrefixToUrl(parts) {
-  if (!parts || parts.length < 2) {
-    return null;
-  }
-  const protocol = parts[0];
-  if (protocol === "file") {
-    let localPath = parts.slice(1).join("/");
-    if (!localPath.startsWith("/")) {
-      localPath = `/${localPath}`;
-    }
-    return `file://${localPath}`;
-  }
-  const host = parts[1];
-  const repoPath = parts.slice(2).join("/");
-  let repoUrl = `${protocol}://${host}`;
-  if (repoPath) {
-    repoUrl += `/${repoPath}`;
-  }
-  if (repoUrl.endsWith("/")) {
-    repoUrl = repoUrl.substring(0, repoUrl.length - 1);
-  }
-  return repoUrl;
-}
-
-// Memoized so that the registry URL and the jar path for one coordinate cost a
-// single lookup between them.
-const _coursierLocationCache = new Map();
-
-/**
- * Locate a Maven coordinate in the local Coursier cache.
- *
- * @param {string} group Maven groupId
- * @param {string} name Maven artifactId (original name with suffix if applicable)
- * @param {string} version Package version
- * @returns {{ repoUrl: string, dir: string }|null} The repository URL and the artifact directory
- */
-function locateInCoursierCache(group, name, version) {
-  if (!group || !name || !version) {
-    return null;
-  }
-  const cacheRoot = coursierCacheRoot();
-  if (!cacheRoot) {
-    return null;
-  }
-  // The cache root is part of the key so that a changed COURSIER_CACHE is never
-  // answered from a memo built against the previous one.
-  const cacheKey = `${cacheRoot}|${group}:${name}:${version}`;
-  if (_coursierLocationCache.has(cacheKey)) {
-    return _coursierLocationCache.get(cacheKey);
-  }
-  const groupParts = group.split(".");
-  const roots = coursierRepoRoots(cacheRoot);
-  let result = null;
-  for (let i = 0; i < roots.length; i++) {
-    const dir = join(cacheRoot, ...roots[i], ...groupParts, name, version);
-    if (safeExistsSync(dir)) {
-      const repoUrl = repoPrefixToUrl(roots[i]);
-      if (repoUrl) {
-        result = { repoUrl, dir };
-        // Most coordinates in a build come from the same handful of
-        // repositories, so keeping the one that just matched at the front makes
-        // the common case a single stat call.
-        if (i > 0) {
-          roots.unshift(roots.splice(i, 1)[0]);
-        }
-        break;
-      }
-    }
-  }
-  _coursierLocationCache.set(cacheKey, result);
-  return result;
-}
-
 /**
  * Find the repository URL from the local Coursier cache for a given Maven package.
  *
@@ -408,6 +249,19 @@ export function findLocalJarPath(group, name, version) {
 // take many minutes and appear as a hang. See
 // https://github.com/cdxgen/cdxgen/issues/4291
 const _jarDistributionCache = new Map();
+registerRunReset("sbt:jar-distributions", () => _jarDistributionCache.clear());
+
+/**
+ * Forget every memoised Coursier lookup, including misses. A long-lived
+ * process calls this per scan so that artifacts downloaded since the previous
+ * scan are found.
+ *
+ * @returns {void}
+ */
+export function resetSbtResolutionCaches() {
+  _jarDistributionCache.clear();
+  resetJvmLocalRepoCaches();
+}
 
 /**
  * Resolve the repo/jar download URLs and optional hashes for a Maven coordinate.
@@ -423,7 +277,7 @@ const _jarDistributionCache = new Map();
  *   Resolved URLs with optional hashes, or null when no registry URL is found.
  */
 export async function resolveJarDistribution(group, name, version) {
-  const cacheKey = `${coursierCacheRoot()}|${group}:${name}:${version}`;
+  const cacheKey = `${coursierCacheDir()}|${group}:${name}:${version}`;
   if (_jarDistributionCache.has(cacheKey)) {
     return _jarDistributionCache.get(cacheKey);
   }
@@ -463,9 +317,12 @@ export async function resolveJarDistribution(group, name, version) {
  * coordinates, building a hierarchical dependency graph. Evicted packages and ranges are ignored.
  *
  * @param {string} sbtTreeFile Path to the sbt dependency tree output file
+ * @param {string} [srcFile] Build definition the tree was resolved from, named as the
+ *   identity source of every component. The tree file itself is a temporary file cdxgen
+ *   deletes after the scan, and naming it would make the BOM differ between two runs.
  * @returns {{ pkgList: Object[], dependenciesList: Object[] }}
  */
-export async function parseSbtTree(sbtTreeFile) {
+export async function parseSbtTree(sbtTreeFile, srcFile = sbtTreeFile) {
   const pkgList = [];
   const dependenciesList = [];
   const keys_cache = {};
@@ -548,12 +405,10 @@ export async function parseSbtTree(sbtTreeFile) {
       continue;
     }
     const originalName = name;
-    const scalaSuffixRegex = /_(2\.\d+|3)$/;
-    const match = name.match(scalaSuffixRegex);
-    let compilerVersion = null;
-    if (match) {
-      compilerVersion = match[1];
-      name = name.replace(scalaSuffixRegex, "");
+    const scalaArtifact = parseScalaArtifact(name);
+    const compilerVersion = scalaArtifact.binaryVersion;
+    if (compilerVersion) {
+      name = scalaArtifact.purlName;
     }
     const distInfo = await resolveJarDistribution(group, originalName, version);
     const qualifiers = { type: "jar" };
@@ -590,7 +445,7 @@ export async function parseSbtTree(sbtTreeFile) {
               {
                 technique: "manifest-analysis",
                 confidence: 1,
-                value: sbtTreeFile,
+                value: srcFile,
               },
             ],
           },
@@ -695,14 +550,9 @@ export async function parseSbtLock(pkgLockFile) {
           }
         }
         const originalName = pkg.name;
-        let name = pkg.name;
-        const scalaSuffixRegex = /_(2\.\d+|3)$/;
-        const match = name.match(scalaSuffixRegex);
-        let compilerVersion = null;
-        if (match) {
-          compilerVersion = match[1];
-          name = name.replace(scalaSuffixRegex, "");
-        }
+        const scalaArtifact = parseScalaArtifact(originalName);
+        const name = scalaArtifact.purlName;
+        const compilerVersion = scalaArtifact.binaryVersion;
         const distInfo = await resolveJarDistribution(
           pkg.org,
           originalName,
@@ -909,6 +759,58 @@ export function parseSbtProjects(stdout) {
   return { projects, root };
 }
 
+/**
+ * Whether an sbt version is sbt 2 or later, the release candidates of sbt 2
+ * included: `2.0.0-RC12` is below `2.0.0` in semver order.
+ *
+ * @param {string|null|undefined} sbtVersion Version from project/build.properties
+ * @returns {boolean} True for sbt 2.x and later
+ */
+export function isSbt2Version(sbtVersion) {
+  return Boolean(sbtVersion && valid(sbtVersion) && major(sbtVersion) >= 2);
+}
+
+/**
+ * Quote a path as an sbt command argument. Inside double quotes sbt reads
+ * backslash escapes, so the backslashes of a Windows path are escaped, and a
+ * path with spaces stays one argument.
+ *
+ * @param {string} filePath Path to quote
+ * @returns {string} The quoted path
+ */
+function sbtQuotedPath(filePath) {
+  return `"${`${filePath}`.replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`;
+}
+
+/**
+ * Build the single command line an sbt 2 dependency tree session runs.
+ *
+ * sbt 2 joins separate command-line arguments into one command line, so the
+ * commands are joined with `;` and passed as one argument. sbt 2 ships the
+ * dependency tree tasks itself, and `dependencyTree / toFile` became
+ * `dependencyTree --out`, so every tree is written to its own file, the
+ * extra roots a tree may list included. The trees are not aggregated: the
+ * tree of an aggregating root would otherwise be overwritten by the trees of
+ * the projects it aggregates. Each tree is scoped to its project, so one
+ * unresolvable subproject cannot fail the others.
+ *
+ * @param {{project?: string, file: string}[]} trees The project id of each
+ *   tree, none for the current project, and the file it is written to
+ * @returns {string} The `;`-joined command line
+ */
+export function sbt2DependencyTreeCommand(trees) {
+  const commands = [
+    "set ThisBuild / asciiGraphWidth := 800",
+    "set ThisBuild / dependencyTree / aggregate := false",
+  ];
+  for (const { project, file } of trees || []) {
+    commands.push(
+      `${project ? `${project}/` : ""}dependencyTree --out ${sbtQuotedPath(file)}`,
+    );
+  }
+  return commands.join("; ");
+}
+
 /**
  * Parse plugins.sbt files to extract sbt plugins as development dependencies.
  *
@@ -970,3 +872,157 @@ export function parseSbtPlugins(projectPath) {
   }
   return plugins;
 }
+
+/**
+ * The sbt launcher jar beside the `sbt.bat` on PATH, or under `SBT_HOME`.
+ * `SBT_LAUNCH_JAR` names one directly.
+ *
+ * @returns {string|undefined} Path of `sbt-launch.jar`
+ */
+export function sbtLaunchJar() {
+  const named = readEnvironmentVariable("SBT_LAUNCH_JAR");
+  if (named && safeExistsSync(named)) {
+    return named;
+  }
+  const dirs =
+    `${readEnvironmentVariable("PATH") || readEnvironmentVariable("Path") || ""}`
+      .split(delimiter)
+      .filter(Boolean);
+  const sbtHome = readEnvironmentVariable("SBT_HOME");
+  if (sbtHome) {
+    dirs.unshift(join(sbtHome, "bin"));
+  }
+  for (const dir of dirs) {
+    if (safeExistsSync(join(dir, "sbt.bat"))) {
+      const jar = join(dir, "sbt-launch.jar");
+      if (safeExistsSync(jar)) {
+        return jar;
+      }
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Split sbt arguments written for a POSIX shell into the arguments the shell
+ * would pass: whitespace separates them outside quotes, and single or double
+ * quotes group words without escapes, so a Windows path keeps its
+ * backslashes.
+ *
+ * @param {string[]} args Shell-quoted argument fragments
+ * @returns {string[]} The arguments
+ */
+export function splitSbtShellArgs(args) {
+  const out = [];
+  for (const fragment of args) {
+    let current = "";
+    let quote;
+    let started = false;
+    for (const ch of `${fragment}`) {
+      if (quote) {
+        if (ch === quote) {
+          quote = undefined;
+        } else {
+          current += ch;
+        }
+      } else if (ch === "'" || ch === '"') {
+        quote = ch;
+        started = true;
+      } else if (/\s/.test(ch)) {
+        if (started) {
+          out.push(current);
+          current = "";
+          started = false;
+        }
+      } else {
+        current += ch;
+        started = true;
+      }
+    }
+    if (started) {
+      out.push(current);
+    }
+  }
+  return out;
+}
+
+/**
+ * The command line that starts sbt from its launcher jar. The options
+ * `sbt.bat` handles itself become what it makes of them: `-no-colors` and
+ * `-D` properties are JVM options, `-addPluginSbtFile` is sbt's own
+ * `--addPluginSbtFile`, and `-batch` and the thin client switches have no
+ * meaning there. The JVM options of `SBT_OPTS`, `JAVA_OPTS` and the build's
+ * `.jvmopts` are kept.
+ *
+ * @param {string[]} args Arguments as given to `sbt`
+ * @param {string} jar The sbt launcher jar
+ * @param {string} [dir] Build directory
+ * @returns {{ command: string, args: string[] }}
+ */
+export function sbtLauncherCommand(args, jar, dir) {
+  const jvm = ["-Dfile.encoding=UTF-8", "-Xss4m", "-Dsbt.supershell=false"];
+  for (const name of ["JAVA_OPTS", "SBT_OPTS"]) {
+    jvm.push(
+      ...`${readEnvironmentVariable(name) || ""}`.split(/\s+/).filter(Boolean),
+    );
+  }
+  const jvmopts = dir ? join(dir, ".jvmopts") : undefined;
+  if (jvmopts && safeExistsSync(jvmopts)) {
+    jvm.push(
+      ...readFileSync(jvmopts, "utf-8")
+        .split(/\s+/)
+        .filter((o) => o && !o.startsWith("#")),
+    );
+  }
+  const commands = [];
+  for (const arg of args) {
+    if (arg === "-no-colors") {
+      jvm.push("-Dsbt.log.noformat=true");
+    } else if (arg.startsWith("-D")) {
+      jvm.push(arg);
+    } else if (arg.startsWith("-J")) {
+      jvm.push(arg.slice(2));
+    } else if (arg.startsWith("-addPluginSbtFile=")) {
+      commands.push(`-${arg}`);
+    } else if (!["-batch", "--server", "--client", "-client"].includes(arg)) {
+      commands.push(arg);
+    }
+  }
+  const javaHome = readEnvironmentVariable("JAVA_HOME");
+  return {
+    command: javaHome ? join(javaHome, "bin", "java.exe") : "java.exe",
+    args: [...jvm, "-jar", jar, ...commands],
+  };
+}
+
+/**
+ * Run sbt. On Windows, `sbt.bat` parses its arguments as batch syntax, which
+ * keeps the POSIX quoting of the commands and mangles quotes inside them, so
+ * sbt starts there from its launcher jar with each command one argument.
+ * Elsewhere, or without a launcher jar, sbt runs as given.
+ *
+ * @param {string} sbtCmd sbt executable
+ * @param {string[]} args sbt arguments
+ * @param {Object} options Spawn options, as for safeSpawnSync
+ * @param {boolean} [shellQuoted] Whether the arguments carry POSIX shell quoting
+ * @returns {Object} The spawn result
+ */
+export function sbtSpawnSync(sbtCmd, args, options, shellQuoted = false) {
+  const jar =
+    isWin && !`${sbtCmd}`.toLowerCase().endsWith(".jar")
+      ? sbtLaunchJar()
+      : undefined;
+  if (!jar) {
+    return safeSpawnSync(sbtCmd, args, options);
+  }
+  const launcher = sbtLauncherCommand(
+    shellQuoted ? splitSbtShellArgs(args) : args,
+    jar,
+    options?.cwd,
+  );
+  const { cdxgenRawShellArgs: _raw, ...spawnOptions } = options || {};
+  return safeSpawnSync(launcher.command, launcher.args, {
+    ...spawnOptions,
+    shell: false,
+  });
+}