@cdxgen/cdxgen 13.3.0 → 13.4.0

package/lib/inventory/buildToolRateLimit.js

@@ -0,0 +1,144 @@
+import { recordDegradation } from "../core/buildLedger.js";
+import { registerRunReset } from "../core/runState.js";
+
+/**
+ * Text a build tool prints when its Maven repository answers HTTP 429, as
+ * recorded from each tool against a repository that rate limits. One small
+ * matcher per tool, plain substring tests on the combined output: the tools
+ * reword their messages between versions, so the matched fragment is the one
+ * every observed phrasing keeps (the status number in the tool's own wording).
+ *
+ * sbt, Mill and scala-cli resolve through Coursier, so they share the
+ * coursier matcher. The Coursier inside sbt 1.x reports the JDK's
+ * `Server returned HTTP response code: 429`; the one in sbt 2, Mill and
+ * scala-cli reports `retryable HTTP error: <url> (HTTP 429)`, with a
+ * `Retry-After` inside the parentheses when the server sent one.
+ *
+ * @type {Object<string, string[]>}
+ */
+const RATE_LIMIT_MARKERS = {
+  maven: ["status code: 429", "Too Many Requests"],
+  gradle: ["Received status code 429", "[HTTP 429: http"],
+  coursier: ["response code: 429", "(HTTP 429"],
+};
+
+/**
+ * The matcher family a tool belongs to.
+ *
+ * @type {Object<string, string>}
+ */
+const TOOL_MATCHER = {
+  maven: "maven",
+  gradle: "gradle",
+  sbt: "coursier",
+  mill: "coursier",
+  "scala-cli": "coursier",
+};
+
+/**
+ * How each tool is moved to a mirror, named in one hint per tool and scan.
+ * sbt reads its own repositories file; Mill and scala-cli read Coursier's
+ * mirror.properties, whose entry spells the semicolon-separated format
+ * because a comma list is silently ignored.
+ *
+ * @type {Object<string, string>}
+ */
+const MIRROR_SETTINGS = {
+  maven:
+    "Point Maven at a mirror with a <mirror> entry in the Maven settings.xml, or set MVNW_REPOURL for the Maven wrapper.",
+  gradle:
+    "Point Gradle at a mirror by declaring the repository in a Gradle init script.",
+  sbt: "Point sbt at a mirror by listing it in ~/.sbt/repositories and passing -Dsbt.override.build.repos=true in SBT_OPTS or .sbtopts.",
+  coursier:
+    "Point Coursier at a mirror with mirror.properties, setting central.from and central.to (separate several central.from URLs with semicolons, not commas).",
+};
+
+/**
+ * Tools whose later invocations or retries cdxgen skips once one met a rate
+ * limit. Mill and scala-cli resolve in a single invocation.
+ *
+ * @type {Set<string>}
+ */
+const STOPPED_TOOLS = new Set(["maven", "gradle", "sbt"]);
+
+/**
+ * Tools whose repository answered HTTP 429 during the current scan. Further
+ * invocations of that tool would only pay the same back-off for the same
+ * empty answer.
+ *
+ * @type {Set<string>}
+ */
+const rateLimitedTools = new Set();
+
+registerRunReset("inventory.buildToolRateLimit", () => {
+  rateLimitedTools.clear();
+});
+
+/**
+ * Whether a tool's combined output names a rate limit from its repository.
+ * The answer text is treated as untrusted: only fixed substrings are looked
+ * for, nothing from the output is kept.
+ *
+ * @param {string} output Combined stdout and stderr of one invocation.
+ * @param {string} tool One of maven, gradle, sbt, mill, scala-cli.
+ * @returns {boolean} True when the output names an HTTP 429 answer.
+ */
+export function detectBuildToolRateLimit(output, tool) {
+  const markers = RATE_LIMIT_MARKERS[TOOL_MATCHER[tool]];
+  if (!markers || typeof output !== "string" || !output) {
+    return false;
+  }
+  return markers.some((marker) => output.includes(marker));
+}
+
+/**
+ * Whether a tool was already found rate limited in this scan, so the caller
+ * can skip its remaining invocations.
+ *
+ * @param {string} tool One of maven, gradle, sbt, mill, scala-cli.
+ * @returns {boolean}
+ */
+export function isBuildToolRateLimited(tool) {
+  return rateLimitedTools.has(tool);
+}
+
+/**
+ * Read a completed invocation's output for a rate limit. When one is found,
+ * the tool is remembered for the scan, one hint naming the tool's mirror
+ * setting is printed, and the `build.rate-limited` degradation is recorded.
+ * The first invocation that shows the rate limit is the one recorded; later
+ * ones add nothing, because the same answer ends them all.
+ *
+ * @param {string} tool One of maven, gradle, sbt, mill, scala-cli.
+ * @param {string|undefined} output Combined output, as commandOutputText
+ *   returns it for a spawn result.
+ * @param {Object} [context] Call-site facts for the degradation event.
+ * @param {string} [context.command] The redacted command that was attempted.
+ * @param {number} [context.exitCode] The invocation's exit status.
+ * @returns {boolean} True when this output names a rate limit.
+ */
+export function noteBuildToolRateLimit(tool, output, context = {}) {
+  if (!detectBuildToolRateLimit(output, tool)) {
+    return false;
+  }
+  if (isBuildToolRateLimited(tool)) {
+    return true;
+  }
+  rateLimitedTools.add(tool);
+  const stopped = STOPPED_TOOLS.has(tool)
+    ? `, so cdxgen runs no further ${tool} resolution in this scan and`
+    : ", so";
+  console.warn(
+    `The repository ${tool} downloads from is rate limiting this machine (HTTP 429)${stopped} its dependency tree is incomplete. ${MIRROR_SETTINGS[tool] || MIRROR_SETTINGS[TOOL_MATCHER[tool]]} cdxgen's own lookups can be moved with MAVEN_CENTRAL_URL.`,
+  );
+  recordDegradation("build.rate-limited", {
+    ecosystem: "java",
+    tool,
+    impact: "transitive-deps",
+    command: context.command,
+    exitCode: context.exitCode,
+    detail: `The repository ${tool} downloads from answered HTTP 429, so ${tool} resolved no dependencies.`,
+    outputExcerpt: output,
+  });
+  return true;
+}

package/lib/inventory/cacheCommands.js

@@ -1,5 +1,6 @@
 /**
- * `cdxgen cache` subcommand — inspect and purge the metadata fetch cache.
+ * `cdxgen cache` subcommand — inspect and purge the metadata fetch cache that
+ * cdxrs and the JS batch pool share.
  *
  * This module operates directly on the filesystem because JS is the authority
  * for the cache directory. The cache layout is:

package/lib/inventory/cmakeBuildDirs.js

@@ -0,0 +1,254 @@
+/**
+ * Where a C/C++ project's build trees are: the build directories its CMake
+ * presets configure, and the conventional ones (`build`, `build-<name>`,
+ * `out`, `builddir`, `cmake-build-<name>`). Shared by the lookups for
+ * `CMakeCache.txt` and for the compilation database, so both search the same
+ * places in the same order.
+ */
+
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { platform } from "node:os";
+import {
+  basename,
+  delimiter,
+  dirname,
+  isAbsolute,
+  join,
+  relative,
+  resolve,
+} from "node:path";
+
+import { isSecureMode, readEnvironmentVariable } from "../core/activity.js";
+import {
+  expandPresetMacros,
+  parseCmakePresets,
+  resolveConfigurePresets,
+} from "../parsers/cmakePresets.js";
+
+/** Presets files CMake reads from the source directory, in order. */
+const PRESETS_FILES = ["CMakePresets.json", "CMakeUserPresets.json"];
+
+/** A presets file larger than this is not read. */
+const MAX_PRESETS_FILE_BYTES = 4 * 1024 * 1024;
+
+/** At most this many presets files are read, includes counted. */
+const MAX_PRESETS_FILES = 64;
+
+/** At most this many build directories are proposed. */
+const MAX_BUILD_DIRS = 128;
+
+const HOST_SYSTEM_NAMES = {
+  darwin: "Darwin",
+  linux: "Linux",
+  win32: "Windows",
+  freebsd: "FreeBSD",
+  openbsd: "OpenBSD",
+  netbsd: "NetBSD",
+  sunos: "SunOS",
+  aix: "AIX",
+};
+
+/**
+ * Whether `child` is `parent` or inside it.
+ *
+ * @param {string} child Path to test
+ * @param {string} parent Directory
+ * @returns {boolean}
+ */
+export function isInsideDir(child, parent) {
+  const rel = relative(resolve(parent), resolve(child));
+  return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel));
+}
+
+function isDirectory(p) {
+  try {
+    return statSync(p).isDirectory();
+  } catch (_err) {
+    return false;
+  }
+}
+
+function readSmallFile(p) {
+  try {
+    const stat = statSync(p);
+    if (!stat.isFile() || stat.size > MAX_PRESETS_FILE_BYTES) {
+      return undefined;
+    }
+    return readFileSync(p, "utf-8");
+  } catch (_err) {
+    return undefined;
+  }
+}
+
+/**
+ * CMake's `${hostSystemName}` for this host.
+ *
+ * @returns {string}
+ */
+export function hostSystemName() {
+  return HOST_SYSTEM_NAMES[platform()] || platform();
+}
+
+/**
+ * Read a project's presets documents: `CMakePresets.json` and
+ * `CMakeUserPresets.json`, each followed by the files it includes (schema
+ * version 4 and later). In secure mode an include outside the project is not
+ * read.
+ *
+ * @param {string} root Project source directory
+ * @returns {Array<{file: string, fileDir: string, document: Object}>}
+ */
+export function readCmakePresetDocuments(root) {
+  const sourceDir = resolve(root);
+  const documents = [];
+  const seen = new Set();
+  const visit = (file) => {
+    const abs = resolve(file);
+    if (seen.has(abs) || seen.size >= MAX_PRESETS_FILES) {
+      return;
+    }
+    seen.add(abs);
+    if (isSecureMode && !isInsideDir(abs, sourceDir)) {
+      return;
+    }
+    const document = parseCmakePresets(readSmallFile(abs));
+    if (!document) {
+      return;
+    }
+    const fileDir = dirname(abs);
+    documents.push({ file: abs, fileDir, document });
+    for (const include of document.include) {
+      const expanded = expandPresetMacros(include, {
+        sourceDir,
+        sourceParentDir: dirname(sourceDir),
+        sourceDirName: basename(sourceDir),
+        fileDir,
+        pathListSep: delimiter,
+        hostSystemName: hostSystemName(),
+        penv: readEnvironmentVariable,
+      });
+      visit(isAbsolute(expanded) ? expanded : join(fileDir, expanded));
+    }
+  };
+  for (const name of PRESETS_FILES) {
+    visit(join(sourceDir, name));
+  }
+  return documents;
+}
+
+/**
+ * The visible configure presets of a project, resolved and with their macros
+ * expanded for this host.
+ *
+ * @param {string} root Project source directory
+ * @returns {Object[]} See `resolveConfigurePresets`
+ */
+export function readCmakeConfigurePresets(root) {
+  const sourceDir = resolve(root);
+  const documents = readCmakePresetDocuments(sourceDir);
+  if (!documents.length) {
+    return [];
+  }
+  return resolveConfigurePresets(documents, {
+    sourceDir,
+    sourceParentDir: dirname(sourceDir),
+    sourceDirName: basename(sourceDir),
+    hostSystemName: hostSystemName(),
+    pathListSep: delimiter,
+    penv: readEnvironmentVariable,
+  });
+}
+
+function childDirs(dir, accept = () => true) {
+  try {
+    return readdirSync(dir, { withFileTypes: true })
+      .filter((e) => e.isDirectory() && accept(e.name))
+      .map((e) => join(dir, e.name))
+      .sort();
+  } catch (_err) {
+    return [];
+  }
+}
+
+/** The order conventional build directories are searched in. */
+function conventionalRank(dir) {
+  const name = basename(dir);
+  if (name === "build") {
+    return 0;
+  }
+  if (name === "out") {
+    return 2;
+  }
+  if (name === "builddir") {
+    return 3;
+  }
+  if (name.startsWith("cmake-build-")) {
+    return 4;
+  }
+  return 1;
+}
+
+/**
+ * The directories a project's build trees may be in, most specific first:
+ * the directory of an explicit `--cmake-cache`, the build directories of its
+ * configure presets, then the `build`, `build-<name>`, `out`, `builddir` and
+ * `cmake-build-<name>` directories at the project root, and the directories
+ * one level below `build`, `out` and `out/build` (where Visual Studio puts
+ * preset builds). Only existing directories are listed; in secure mode, only
+ * those inside the project.
+ *
+ * @param {string} root Project source directory
+ * @param {Object} [options] CLI options (`cmakeCache`)
+ * @returns {string[]} Absolute directories, without duplicates
+ */
+export function cmakeBuildDirCandidates(root, options = {}) {
+  const sourceDir = resolve(root);
+  const dirs = [];
+  const add = (d) => {
+    if (!d || dirs.length >= MAX_BUILD_DIRS) {
+      return;
+    }
+    const abs = resolve(d);
+    if (dirs.includes(abs) || abs === sourceDir) {
+      return;
+    }
+    if (isSecureMode && !isInsideDir(abs, sourceDir)) {
+      return;
+    }
+    if (isDirectory(abs)) {
+      dirs.push(abs);
+    }
+  };
+  if (options.cmakeCache) {
+    add(dirname(resolve(`${options.cmakeCache}`)));
+  }
+  if (!isDirectory(sourceDir)) {
+    return dirs;
+  }
+  for (const preset of readCmakeConfigurePresets(sourceDir)) {
+    add(preset.binaryDir);
+  }
+  const rootBuildDirs = childDirs(
+    sourceDir,
+    (name) =>
+      name.startsWith("build") ||
+      name === "out" ||
+      name.startsWith("cmake-build-"),
+  );
+  rootBuildDirs.sort(
+    (a, b) => conventionalRank(a) - conventionalRank(b) || a.localeCompare(b),
+  );
+  for (const d of rootBuildDirs) {
+    add(d);
+  }
+  for (const parent of [
+    join(sourceDir, "build"),
+    join(sourceDir, "out"),
+    join(sourceDir, "out", "build"),
+  ]) {
+    for (const d of childDirs(parent)) {
+      add(d);
+    }
+  }
+  return dirs;
+}