@cdxgen/cdxgen 13.2.0 → 13.4.0

package/lib/ecosystems/golem.js

@@ -11,6 +11,10 @@ import {
   safeSpawnSync,
 } from "../core/fs.js";
 import { dirNameStr } from "../core/paths.js";
+import {
+  reportPluginRunFailure,
+  resolvePluginSourceDir,
+} from "../core/pluginRun.js";
 import { resolvePluginBinary } from "../inventory/plugins.js";
 
 const GO_LANGUAGES = new Set(["go", "golang"]);
@@ -1342,10 +1346,13 @@ function normalizedGolemDataFlowMode(options = {}) {
   return dataFlowMode;
 }
 
-function normalizedGolemCallGraphMode(dataFlowMode, options = {}) {
+function normalizedGolemCallGraphMode(options = {}) {
   let callgraphMode = String(options.golemCallgraph || "static").toLowerCase();
   if (!GOLEM_CALLGRAPH_MODES.has(callgraphMode)) {
-    callgraphMode = dataFlowMode || "static";
+    // An unrecognised mode falls back to the default. It used to fall back to
+    // the data-flow mode ("security", "all"), which is not a call-graph mode
+    // and made golem reject the run.
+    callgraphMode = "static";
   }
   return callgraphMode;
 }
@@ -1637,11 +1644,17 @@ export function runGolemAnalysis(src, outputFile, options = {}) {
     return false;
   }
   const dataFlowMode = normalizedGolemDataFlowMode(options);
-  const callgraphMode = normalizedGolemCallGraphMode(dataFlowMode, options);
+  const callgraphMode = normalizedGolemCallGraphMode(options);
+  // cdxgen may run with its working directory already inside the project
+  // while the source argument is relative to a repository root; a plain
+  // resolve would join the two into a directory that does not exist. The
+  // helper checks the result and falls back to the working directory when it
+  // is itself the Go module under analysis.
+  const analysisDir = resolvePluginSourceDir(src, ["go.mod", "go.work"]);
   const args = [
     "analyze",
     "--dir",
-    resolve(src),
+    analysisDir,
     "--format",
     "json",
     "--callgraph",
@@ -1666,17 +1679,20 @@ export function runGolemAnalysis(src, outputFile, options = {}) {
     console.log("Executing", executable, args.join(" "));
   }
   const result = safeSpawnSync(executable, args, {
-    cwd: resolve(src),
+    cwd: analysisDir,
     shell: false,
   });
   if (result?.status !== 0 || result?.error || !safeExistsSync(outputFile)) {
-    if (DEBUG_MODE) {
-      if (result?.stdout || result?.stderr) {
-        console.error(result.stdout, result.stderr);
-      } else {
-        console.log("Check if the golem plugin was installed successfully.");
-      }
-    }
+    reportPluginRunFailure({
+      tool: "golem",
+      ecosystem: "go",
+      executable,
+      args,
+      dir: analysisDir,
+      result,
+      outputFile,
+      options,
+    });
     return false;
   }
   return true;

package/lib/ecosystems/gradleutils.js

@@ -1,10 +1,11 @@
 import { Buffer } from "node:buffer";
-import { chmodSync, readFileSync } from "node:fs";
+import { chmodSync, readdirSync, readFileSync } from "node:fs";
 import { homedir, platform } from "node:os";
 import { basename, join, resolve } from "node:path";
+import process from "node:process";
 
 import { build } from "@cdxgen/cdx-purl";
-import * as toml from "smol-toml";
+import { valid } from "semver";
 
 import {
   DEBUG_MODE,
@@ -22,6 +23,11 @@ import {
 import { getAllFiles, safeExistsSync, safeSpawnSync } from "../core/fs.js";
 import { thoughtLog } from "../core/logger.js";
 import { isWin } from "../core/paths.js";
+import { parseToml } from "../core/toml.js";
+import {
+  isBuildToolRateLimited,
+  noteBuildToolRateLimit,
+} from "../inventory/buildToolRateLimit.js";
 import { collectJarNS } from "../inventory/deps.js";
 import { minimumGradleVersionForJava } from "../inventory/jvmToolEnv.js";
 import { applyPurl, tryBuildPurl } from "../inventory/purl.js";
@@ -41,19 +47,33 @@ function isJvmToolPinned() {
 /**
  * Method to return the gradle command to use.
  *
+ * Project-local wrapper scripts (`gradlew`) are attacker-controlled files in
+ * the scanned project, so they are only used when dependency tooling is
+ * allowed (`options.installDeps !== false`); otherwise the system Gradle is
+ * used.
+ *
  * @param {string} srcPath Path to look for gradlew wrapper
  * @param {string|null} rootPath Root directory to look for gradlew wrapper
+ * @param {Object} [options] CLI options (`installDeps` gates wrapper use)
  */
-export function getGradleCommand(srcPath, rootPath) {
+export function getGradleCommand(srcPath, rootPath, options = {}) {
   let gradleCmd = "gradle";
   const pinnedToolSelected = isJvmToolPinned();
+  // Introspection mode exists to observe the project's own toolchain, so it
+  // keeps using wrappers even when dependency installation is disabled.
+  const allowProjectWrappers =
+    options.installDeps !== false || options.introspect === true;
 
   let findGradleFile = "gradlew";
   if (platform() === "win32") {
     findGradleFile = "gradlew.bat";
   }
 
-  if (!pinnedToolSelected && safeExistsSync(join(srcPath, findGradleFile))) {
+  if (
+    allowProjectWrappers &&
+    !pinnedToolSelected &&
+    safeExistsSync(join(srcPath, findGradleFile))
+  ) {
     // Use local gradle wrapper if available
     // Enable execute permission
     try {
@@ -72,6 +92,7 @@ export function getGradleCommand(srcPath, rootPath) {
       reason: `Selected project-local Gradle wrapper ${gradleCmd}.`,
     });
   } else if (
+    allowProjectWrappers &&
     !pinnedToolSelected &&
     rootPath &&
     safeExistsSync(join(rootPath, findGradleFile))
@@ -137,6 +158,65 @@ export function getGradleCommand(srcPath, rootPath) {
   return gradleCmd;
 }
 
+/**
+ * System properties that force the evaluation model cdxgen's Gradle
+ * introspection depends on.
+ *
+ * The bundled init script registers `afterEvaluate` on every project and a
+ * `taskGraph.whenReady` listener, so it cannot run under isolated projects
+ * (which forbid reaching into another project's model and fail the build at
+ * configuration time, see issue #4444) nor under the configuration cache (whose
+ * reuse skips init scripts entirely, dropping the `<CDXGEN:repository>` and
+ * `<CDXGEN:includedBuild>` markers even when the first run succeeded). Both
+ * features are commonly enabled from `gradle.properties` (e.g. okhttp), which
+ * command-line `-D` flags override, so every cdxgen invocation turns them
+ * off: these are ephemeral introspection builds, not the user's build. The
+ * isolated-projects property is spelled `org.gradle.unsafe.isolated-projects`
+ * from Gradle 8.6 until Gradle 9.7 renamed it, so both spellings are forced
+ * off to cover every Gradle in between. The configuration cache likewise
+ * keeps its pre-8.1 `org.gradle.unsafe.configuration-cache` spelling.
+ *
+ * Each feature is handled as one group. Gradle disables a feature when any of
+ * its spellings is explicitly `false`, so if the user already chose a value
+ * via `GRADLE_ARGS`, either as a `-D` property under any spelling or as the
+ * `--[no-]isolated-projects` / `--[no-]configuration-cache` option (which
+ * beats `-D` properties), cdxgen adds no override for that feature at all.
+ *
+ * @param {string[]} gradleArguments User-supplied general gradle arguments.
+ * @returns {string[]} The `-D` flags to inject before the user's arguments.
+ */
+function gradleEvaluationOverrides(gradleArguments) {
+  const features = [
+    {
+      option: "isolated-projects",
+      properties: [
+        "org.gradle.isolated-projects",
+        "org.gradle.unsafe.isolated-projects",
+      ],
+    },
+    {
+      option: "configuration-cache",
+      properties: [
+        "org.gradle.configuration-cache",
+        "org.gradle.unsafe.configuration-cache",
+      ],
+    },
+  ];
+  const overrides = [];
+  for (const { option, properties } of features) {
+    const userChose = gradleArguments.some(
+      (arg) =>
+        arg === `--${option}` ||
+        arg === `--no-${option}` ||
+        properties.some((property) => arg.startsWith(`-D${property}=`)),
+    );
+    if (!userChose) {
+      overrides.push(...properties.map((property) => `-D${property}=false`));
+    }
+  }
+  return overrides;
+}
+
 /**
  * Method to combine the general gradle arguments, the sub-commands and the sub-commands' arguments in the correct way
  *
@@ -160,6 +240,7 @@ export function buildGradleCommandArguments(
     "--no-parallel",
   ]
     .concat(getGradleDaemonParameter())
+    .concat(gradleEvaluationOverrides(gradleArguments))
     .concat(gradleArguments);
   const maxCliArgsLength = isWin
     ? 7500 - gradleCommandLength - mainGradleArguments.join(" ").length - 2
@@ -202,47 +283,87 @@ function getGradleDaemonParameter() {
   }
 }
 
+/**
+ * Parse a Gradle task header line such as `> Task :app:dependencies FAILED`
+ * into the path of the project that owns the task and the task's name.
+ *
+ * @param {string} line A line of Gradle output
+ * @returns {{projectPath: string, taskName: string}|undefined} The owning
+ *   project's path (`""` for a root task such as `:dependencies`) and the task
+ *   name, or undefined when the line is not a task header
+ */
+function parseGradleTaskHeader(line) {
+  if (!line.startsWith("> Task :")) {
+    return undefined;
+  }
+  // The path is the first token: a status such as UP-TO-DATE or FAILED may
+  // follow it, and a CRLF line keeps its carriage return.
+  const taskPath = line.slice("> Task ".length).trim().split(/\s+/)[0];
+  const lastColon = taskPath.lastIndexOf(":");
+  return {
+    projectPath: taskPath.slice(0, lastColon),
+    taskName: taskPath.slice(lastColon + 1),
+  };
+}
+
 /**
  * Method to split the output produced by Gradle using parallel processing by project
  *
+ * Gradle groups console output by task and prints a `> Task :path` header
+ * before each group, repeating it whenever a task's output resumes after
+ * another task's. Each group is therefore keyed by the project in its own
+ * header (`> Task :app:dependencies` belongs to `:app`), never by whatever
+ * report header happened to precede it, and the groups of one project are
+ * concatenated in order, so reordering or interrupting the task reports cannot
+ * move a report to another project or overwrite one (issue 4465). The groups
+ * of a root task (`:dependencies`) all take the root project's name from the
+ * `Root project '…'` header that only a root task prints, so a continuation
+ * group, which carries no report header, stays with the rest of the root
+ * report. The groups of other tasks are dropped along with their headers.
+ *
  * @param {string} rawOutput Full output produced by Gradle using parallel processing
  * @param {string[]} relevantTasks The list of gradle tasks whose output need to be considered.
  * @returns {map} Map with subProject names as keys and corresponding dependency task outputs as values.
  */
 export function splitOutputByGradleProjects(rawOutput, relevantTasks) {
   const outputSplitBySubprojects = new Map();
-  let subProjectOut = "";
-  const outSplitByLine = rawOutput.split("\n");
-  let currentProjectName = "root";
-  const regexPatternForRelevantTasks = `.*:(${relevantTasks.join("|")})(?=s|\r|$)`;
-  const regexForRelevantTasks = new RegExp(regexPatternForRelevantTasks);
-  for (const [i, line] of outSplitByLine.entries()) {
-    //filter out everything before first task output
-    if (!line.startsWith("> Task :") && subProjectOut === "") {
-      continue;
+  const relevantTaskNames = new Set(relevantTasks);
+  const taskChunks = [];
+  // Lines before the first task header (build logs) belong to no task.
+  let currentChunk;
+  for (const line of rawOutput.split("\n")) {
+    const taskHeader = parseGradleTaskHeader(line);
+    if (taskHeader) {
+      currentChunk = relevantTaskNames.has(taskHeader.taskName)
+        ? { projectPath: taskHeader.projectPath, lines: [line] }
+        : undefined;
+      if (currentChunk) {
+        taskChunks.push(currentChunk);
+      }
+    } else if (currentChunk) {
+      currentChunk.lines.push(line);
     }
-
-    //ignore output of irrelevant tasks
-    if (line.startsWith("> Task :") && !regexForRelevantTasks.test(line)) {
+  }
+  let rootProjectName;
+  for (const chunk of taskChunks) {
+    if (chunk.projectPath) {
       continue;
     }
-
-    if (line.startsWith("Root project '") || line.startsWith("Project ':")) {
-      currentProjectName = line.split("'")[1];
-      outputSplitBySubprojects.set(currentProjectName, "");
-    }
-    // if previous subProject has ended, push to array and reset subProject string
-    if (line.startsWith("> Task :") && subProjectOut !== "") {
-      outputSplitBySubprojects.set(currentProjectName, subProjectOut);
-      subProjectOut = "";
-    }
-    //if in subproject block, keep appending to string
-    subProjectOut += `${line}\n`;
-    //if end of last dependencies output block, push to array
-    if (i === outSplitByLine.length - 1) {
-      outputSplitBySubprojects.set(currentProjectName, subProjectOut);
+    const rootHeader = chunk.lines.find((line) =>
+      line.startsWith("Root project '"),
+    );
+    if (rootHeader) {
+      rootProjectName = rootHeader.split("'")[1];
+      break;
     }
   }
+  for (const chunk of taskChunks) {
+    const projectName = chunk.projectPath || rootProjectName || "root";
+    outputSplitBySubprojects.set(
+      projectName,
+      `${outputSplitBySubprojects.get(projectName) || ""}${chunk.lines.join("\n")}\n`,
+    );
+  }
   return outputSplitBySubprojects;
 }
 
@@ -380,8 +501,14 @@ export function executeParallelGradleProperties(
   dir,
   allProjectsStr,
   extraArgs = [],
+  options = {},
 ) {
-  const gradleCmd = getGradleCommand(dir, null);
+  // This runs once per properties round, so a rate limit an earlier round met
+  // ends the later rounds too.
+  if (isBuildToolRateLimited("gradle")) {
+    return "";
+  }
+  const gradleCmd = getGradleCommand(dir, null, options);
   const gradleArgs = buildGradleCommandArguments(
     extraArgs.concat(
       readEnvironmentVariable("GRADLE_ARGS")
@@ -407,6 +534,22 @@ export function executeParallelGradleProperties(
       cwd: dir,
       shell: isWin,
     });
+    // A properties invocation needs the same repository the build does, so a
+    // 429 answer here ends every further gradle invocation in this scan. The
+    // dependencies task can even exit 0 under a rate limit, so the output is
+    // read whatever the status is. The properties this invocation did print
+    // are kept.
+    if (
+      noteBuildToolRateLimit("gradle", commandOutputText(result), {
+        command: `${basename(gradleCmd)} ${gradleArg.join(" ")}`,
+        exitCode: typeof result.status === "number" ? result.status : undefined,
+      })
+    ) {
+      if (result.stdout) {
+        allOutputs.push(result.stdout);
+      }
+      break;
+    }
     if (result.status !== 0 || result.error) {
       recordGradleInvocationFailure(result, {
         command: `${basename(gradleCmd)} ${gradleArg.join(" ")}`,
@@ -1205,11 +1348,16 @@ export async function collectGradleDependencies(
 /**
  * Method to return the mill command to use.
  *
+ * The project-local `mill` script is an attacker-controlled file in the
+ * scanned project, so it is only used when dependency tooling is allowed
+ * (`options.installDeps !== false`).
+ *
  * @param {string} srcPath Path to look for mill wrapper
+ * @param {Object} [options] CLI options (`installDeps` gates wrapper use)
  */
-export function getMillCommand(srcPath) {
+export function getMillCommand(srcPath, options = {}) {
   let millCmd = `mill${platform() === "win32" ? ".bat" : ""}`;
-  if (safeExistsSync(join(srcPath, millCmd))) {
+  if (options.installDeps !== false && safeExistsSync(join(srcPath, millCmd))) {
     // Use local mill wrapper if available
     // Enable execute permission
     try {
@@ -1223,18 +1371,184 @@ export function getMillCommand(srcPath) {
   return millCmd;
 }
 
+/**
+ * Determine the Mill version of a build.
+ *
+ * The version is read from the `.mill-version` file, from the
+ * `//| mill-version:` header of the build file, or from the launcher itself.
+ *
+ * @param {string} millRootPath Root of the Mill build
+ * @param {string} millCmd Mill command to use for the fallback probe
+ * @returns {string|null} The Mill version, or null when it cannot be determined
+ */
+export function getMillVersion(millRootPath, millCmd) {
+  const millVersionFile = join(millRootPath, ".mill-version");
+  if (safeExistsSync(millVersionFile)) {
+    const version = (readFileSync(millVersionFile, "utf-8") || "").trim();
+    if (valid(version)) {
+      return version;
+    }
+  }
+  for (const buildFile of ["build.mill", "build.sc"]) {
+    const buildFilePath = join(millRootPath, buildFile);
+    if (!safeExistsSync(buildFilePath)) {
+      continue;
+    }
+    const headerMatch = /^\/\/[|\s]\s*mill-version:\s*(\S+)/m.exec(
+      readFileSync(buildFilePath, "utf-8"),
+    );
+    if (headerMatch?.[1] && valid(headerMatch[1])) {
+      return headerMatch[1];
+    }
+  }
+  const versionResult = safeSpawnSync(millCmd, ["--no-server", "--version"], {
+    cwd: millRootPath,
+    shell: isWin,
+  });
+  const versionMatch = /version\s+(\S+)/.exec(`${versionResult.stdout || ""}`);
+  return versionMatch?.[1] && valid(versionMatch[1]) ? versionMatch[1] : null;
+}
+
+/**
+ * Find the Mill daemons running under a tree.
+ *
+ * Daemons record their process id in `out/mill-daemon*` (Mill 1.x) or
+ * `out/mill-server*` directories, directly or one level below (Mill 0.12
+ * keeps one subdirectory per server). Only processes that are still alive
+ * are returned.
+ *
+ * @param {string} rootPath Tree to search
+ * @returns {Map<string, Set<number>>} Live daemon process ids by the root of
+ *   the build they belong to
+ */
+export function findMillDaemons(rootPath) {
+  const daemons = new Map();
+  const readPid = (dir) => {
+    for (const candidate of [dir, ...safeSubdirs(dir)]) {
+      try {
+        const pid = Number.parseInt(
+          readFileSync(join(candidate, "processId"), "utf-8").trim(),
+          10,
+        );
+        if (pid > 0 && isProcessAlive(pid)) {
+          return pid;
+        }
+      } catch (_err) {
+        // no process id recorded in this directory
+      }
+    }
+    return undefined;
+  };
+  const visit = (dir, depth) => {
+    if (depth > 4) {
+      return;
+    }
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch (_err) {
+      return;
+    }
+    for (const entry of entries) {
+      if (!entry.isDirectory()) {
+        continue;
+      }
+      if (entry.name === "out") {
+        for (const name of safeSubdirs(join(dir, entry.name))) {
+          const base = basename(name);
+          if (
+            !base.startsWith("mill-daemon") &&
+            !base.startsWith("mill-server")
+          ) {
+            continue;
+          }
+          const pid = readPid(name);
+          if (pid) {
+            if (!daemons.has(dir)) {
+              daemons.set(dir, new Set());
+            }
+            daemons.get(dir).add(pid);
+          }
+        }
+      } else if (!["target", "node_modules", ".git"].includes(entry.name)) {
+        visit(join(dir, entry.name), depth + 1);
+      }
+    }
+  };
+  visit(rootPath, 0);
+  return daemons;
+}
+
+function safeSubdirs(dir) {
+  try {
+    return readdirSync(dir, { withFileTypes: true })
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => join(dir, entry.name));
+  } catch (_err) {
+    return [];
+  }
+}
+
+function isProcessAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    return err?.code === "EPERM";
+  }
+}
+
+/**
+ * Stop the Mill daemons that started under the scanned tree during the scan.
+ *
+ * cdxgen's own Mill calls run with --no-server, but a project build cdxgen
+ * runs can shell out to Mill without it and leave a daemon behind. Daemons
+ * that were already running when the scan started belong to the user, for
+ * example to an IDE, and are left alone; every other live daemon is shut
+ * down through the launcher of the build it belongs to.
+ *
+ * @param {string} rootPath Scanned tree
+ * @param {Map<string, Set<number>>} [runningBefore] Result of
+ *   {@link findMillDaemons} taken before the scan
+ */
+export function stopMillDaemons(rootPath, runningBefore = new Map()) {
+  for (const [buildRoot, pids] of findMillDaemons(rootPath)) {
+    const before = runningBefore.get(buildRoot) || new Set();
+    if ([...pids].every((pid) => before.has(pid))) {
+      continue;
+    }
+    const launcher = safeExistsSync(join(buildRoot, "mill"))
+      ? join(buildRoot, "mill")
+      : "mill";
+    console.log("Stopping the Mill daemon the scan started in", buildRoot);
+    safeSpawnSync(launcher, ["shutdown"], {
+      cwd: buildRoot,
+      shell: isWin,
+    });
+  }
+}
+
 /**
  * Method to return the maven command to use.
  *
+ * Project-local wrapper scripts (`mvnw`) are attacker-controlled files in the
+ * scanned project, so they are only used when dependency tooling is allowed
+ * (`options.installDeps !== false`); otherwise the installed Maven is used.
+ *
  * @param {string} srcPath Path to look for maven wrapper
  * @param {string} rootPath Root directory to look for maven wrapper
+ * @param {Object} [options] CLI options (`installDeps` gates wrapper use)
  */
-export function getMavenCommand(srcPath, rootPath) {
+export function getMavenCommand(srcPath, rootPath, options = {}) {
   let mavenCmd = "mvn";
   // Check if the wrapper script is both available and functional
   let isWrapperReady = false;
   let isWrapperFound = false;
   const pinnedToolSelected = isJvmToolPinned();
+  // Introspection mode exists to observe the project's own toolchain, so it
+  // keeps using wrappers even when dependency installation is disabled.
+  const allowProjectWrappers =
+    options.installDeps !== false || options.introspect === true;
   let findMavenFile = "mvnw";
   let mavenWrapperCmd = null;
   if (platform() === "win32") {
@@ -1247,7 +1561,11 @@ export function getMavenCommand(srcPath, rootPath) {
     }
   }
 
-  if (!pinnedToolSelected && safeExistsSync(join(srcPath, findMavenFile))) {
+  if (
+    allowProjectWrappers &&
+    !pinnedToolSelected &&
+    safeExistsSync(join(srcPath, findMavenFile))
+  ) {
     // Use local maven wrapper if available
     // Enable execute permission
     try {
@@ -1267,6 +1585,7 @@ export function getMavenCommand(srcPath, rootPath) {
       reason: `Found Maven wrapper candidate ${mavenWrapperCmd}.`,
     });
   } else if (
+    allowProjectWrappers &&
     !pinnedToolSelected &&
     rootPath &&
     safeExistsSync(join(rootPath, findMavenFile))
@@ -1410,7 +1729,7 @@ export function getMavenCommand(srcPath, rootPath) {
 export function parseGradleVersionCatalog(catalogFile) {
   let catalog;
   try {
-    catalog = toml.parse(readFileSync(catalogFile, "utf-8"));
+    catalog = parseToml(readFileSync(catalogFile, "utf-8"));
   } catch (error) {
     console.warn(`Failed to parse ${catalogFile}: ${error.message}`);
     return [];