@cdxgen/cdxgen 13.5.0 → 13.6.0

package/lib/inventory/atomUtils.js

@@ -183,6 +183,7 @@ export function findCompileCommands(src, options = {}) {
 }
 
 const atomFrontendArgKeysCache = new Map();
+const atomOptionsCache = new Map();
 
 /**
  * Split an atom command that may carry a leading argument (`node
@@ -271,6 +272,43 @@ export function atomFrontendArgKeys(language) {
   return keys;
 }
 
+/**
+ * The options the installed atom accepts, as `atom --help` lists them. Asked
+ * once per atom command. An atom that fails to start accepts none.
+ *
+ * @returns {Set<string>} Supported options, such as `--bom-file`
+ */
+export function atomOptions() {
+  const command = getAtomCommand();
+  if (atomOptionsCache.has(command)) {
+    return atomOptionsCache.get(command);
+  }
+  const [bin, leadingArg] = splitAtomCommand(command);
+  const args = ["--help"];
+  if (leadingArg !== undefined) {
+    args.unshift(leadingArg);
+  }
+  const env = { ...process.env };
+  if (readEnvironmentVariable("ATOM_JAVA_HOME")) {
+    env.JAVA_HOME = readEnvironmentVariable("ATOM_JAVA_HOME");
+  }
+  const options = new Set();
+  const result = safeSpawnSync(bin, args, {
+    shell: isWin,
+    env,
+    timeout: 120000,
+  });
+  if (result?.status === 0 && result.stdout) {
+    for (const match of `${result.stdout}`.matchAll(
+      /(?:^|\s)(--[a-z][a-z0-9-]*)/g,
+    )) {
+      options.add(match[1]);
+    }
+  }
+  atomOptionsCache.set(command, options);
+  return options;
+}
+
 /**
  * The `--frontend-args` atom takes for a C/C++ language: the project's
  * compilation database, when there is one and the installed atom reads it
@@ -315,6 +353,45 @@ export function atomCompileCommandsArgs(src, language, options = {}) {
   return ["--frontend-args", `compile-commands=${database}`];
 }
 
+/**
+ * The `--bom-file` a reachables slice takes package purls from: the BOM evinse
+ * annotates. atom would otherwise only find a BOM saved as `bom.json` inside
+ * the scanned directory, so a BOM written anywhere else gave every reachable
+ * flow no purl. An atom without `--bom-file` is given the arguments it has
+ * always been given, and still reads a `bom.json` in the scanned directory.
+ *
+ * @param {string} sliceType atom slice command
+ * @param {Object} options CLI options, for `input`
+ * @returns {string[]} Arguments to add to the atom command
+ */
+export function atomBomFileArgs(sliceType, options = {}) {
+  if (
+    sliceType !== "reachables" ||
+    !options?.input ||
+    !safeExistsSync(options.input)
+  ) {
+    return [];
+  }
+  const bomFile = resolve(options.input);
+  // The argument reaches a Windows shell command line through `shell: isWin`
+  // in executeAtom, so a path a shell would interpret is never passed on.
+  if (UNSAFE_SHELL_CHARS_RE.test(bomFile)) {
+    console.warn(
+      `The BOM path ${bomFile} contains characters a shell would interpret. Reachable flows will carry purls only if a bom.json is in the scanned directory.`,
+    );
+    return [];
+  }
+  if (!atomOptions().has("--bom-file")) {
+    if (DEBUG_MODE) {
+      console.log(
+        "The installed atom does not take --bom-file. Reachable flows will carry purls only if a bom.json is in the scanned directory.",
+      );
+    }
+    return [];
+  }
+  return ["--bom-file", bomFile];
+}
+
 /**
  * The arguments that stop atom's Java frontend from running the scanned
  * project's build. In its default delombok mode, atom runs

package/lib/inventory/brotliDecoder.js

@@ -0,0 +1,142 @@
+// BOM components for the Brotli decoders compiled into native files. The
+// detection itself, and why its markers can be trusted, is in
+// lib/core/brotliProbe.js.
+
+import {
+  MAX_BROTLI_PROBE_BYTES,
+  MAX_BROTLI_PROBES,
+  readBrotliDecoder,
+} from "../core/brotliProbe.js";
+import { applyPurl, tryBuildPurl } from "./purl.js";
+
+export { MAX_BROTLI_PROBE_BYTES, MAX_BROTLI_PROBES, readBrotliDecoder };
+
+/**
+ * The properties a file component carries once it has been probed: the fix
+ * state of each decoder found in it, or why it could not be read.
+ *
+ * @param {Object} decoder result of readBrotliDecoder
+ * @returns {{name: string, value: string}[]} properties to append
+ */
+export function brotliDecoderProperties(decoder) {
+  const properties = [];
+  if (decoder?.skipped) {
+    properties.push({ name: "cdx:brotli:probe", value: decoder.skipped });
+  }
+  if (decoder?.c) {
+    properties.push(
+      { name: "cdx:brotli:c:blockSwitchFix", value: decoder.c.blockSwitchFix },
+      { name: "cdx:brotli:c:evidence", value: decoder.c.evidence },
+    );
+  }
+  if (decoder?.go) {
+    properties.push({
+      name: "cdx:brotli:go:blockSwitchFix",
+      value: decoder.go.blockSwitchFix,
+    });
+  }
+  return properties;
+}
+
+/**
+ * The library component every embedded decoder links to. It has no version:
+ * no build records one, and only the per-file cdx:brotli:*:blockSwitchFix
+ * properties say whether a copy is vulnerable. The `vcs_url` qualifier names
+ * the project and keeps the identity apart from an unpackaged `brotli`
+ * executable, which is `pkg:generic/brotli`. The purl has no namespace: a
+ * `pkg:github` purl is typed as an application and `pkg:generic/google` as a
+ * framework when the BOM is assembled, and this is neither.
+ *
+ * @returns {Object} component with `purl` and `bom-ref`
+ */
+export function createBrotliComponent() {
+  return applyPurl(
+    {
+      type: "library",
+      name: "brotli",
+      description:
+        "Brotli decoder compiled into native files; see the cdx:brotli properties on each file",
+      properties: [],
+    },
+    tryBuildPurl({
+      type: "generic",
+      name: "brotli",
+      qualifiers: { vcs_url: "git+https://github.com/google/brotli" },
+    }),
+  );
+}
+
+/**
+ * Start collecting the decoder copies of one scan.
+ *
+ * @returns {{component: Object|undefined, dependencies: Object[], copies: number, unpatched: number, unknown: number}}
+ */
+export function createBrotliInventory() {
+  return {
+    component: undefined,
+    dependencies: [],
+    copies: 0,
+    unpatched: 0,
+    unknown: 0,
+  };
+}
+
+/**
+ * Tag a probed file component and, when it holds a decoder, link it to the
+ * shared brotli component, creating that component on first use. Mutates the
+ * file component in place.
+ *
+ * @param {Object} inventory accumulator from createBrotliInventory
+ * @param {Object} fileComponent file component with a `properties` array
+ * @param {Object} decoder result of readBrotliDecoder
+ */
+export function attachBrotliDecoder(inventory, fileComponent, decoder) {
+  const properties = brotliDecoderProperties(decoder);
+  if (!properties.length) {
+    return;
+  }
+  fileComponent.properties.push(...properties);
+  if (!decoder.c && !decoder.go) {
+    return;
+  }
+  if (!inventory.component) {
+    inventory.component = createBrotliComponent();
+  }
+  const states = [decoder.c?.blockSwitchFix, decoder.go?.blockSwitchFix];
+  inventory.copies += 1;
+  if (states.includes("false")) {
+    inventory.unpatched += 1;
+  } else if (states.includes("unknown")) {
+    inventory.unknown += 1;
+  }
+  if (fileComponent["bom-ref"] && inventory.component["bom-ref"]) {
+    inventory.dependencies.push({
+      ref: fileComponent["bom-ref"],
+      dependsOn: [inventory.component["bom-ref"]],
+    });
+  }
+}
+
+/**
+ * Finish an inventory: record the copy counts on the brotli component.
+ *
+ * @param {Object} inventory accumulator from createBrotliInventory
+ * @returns {{components: Object[], dependencies: Object[]}} what to add to the BOM
+ */
+export function finishBrotliInventory(inventory) {
+  if (!inventory?.component) {
+    return { components: [], dependencies: [] };
+  }
+  inventory.component.properties = [
+    { name: "cdx:brotli:embeddedCopies", value: String(inventory.copies) },
+    { name: "cdx:brotli:unpatchedCopies", value: String(inventory.unpatched) },
+    { name: "cdx:brotli:unknownCopies", value: String(inventory.unknown) },
+  ];
+  return {
+    components: [inventory.component],
+    dependencies: [
+      { ref: inventory.component["bom-ref"], dependsOn: [] },
+      ...inventory.dependencies,
+    ],
+  };
+}

package/lib/inventory/buildToolRateLimit.js

@@ -129,7 +129,7 @@ export function noteBuildToolRateLimit(tool, output, context = {}) {
     ? `, so cdxgen runs no further ${tool} resolution in this scan and`
     : ", so";
   console.warn(
-    `The repository ${tool} downloads from is rate limiting this machine (HTTP 429)${stopped} its dependency tree is incomplete. ${MIRROR_SETTINGS[tool] || MIRROR_SETTINGS[TOOL_MATCHER[tool]]} cdxgen's own lookups can be moved with MAVEN_CENTRAL_URL.`,
+    `The repository ${tool} downloads from is rate limiting this machine (HTTP 429)${stopped} its dependency tree is incomplete. ${MIRROR_SETTINGS[tool] || MIRROR_SETTINGS[TOOL_MATCHER[tool]]} Set MAVEN_CENTRAL_MIRROR=true to point every build tool at Google's Maven Central mirror, or MAVEN_CENTRAL_URL for cdxgen's own lookups.`,
   );
   recordDegradation("build.rate-limited", {
     ecosystem: "java",

package/lib/inventory/cbomutils.js

@@ -601,19 +601,68 @@ export async function collectDosaiCryptoComponents(src, options = {}) {
 /**
  * Find crypto algorithm in the given code snippet
  *
+ * cleanStr fuses the snippet's identifiers into one lowercase string
+ * (`prepare_for_save(iv` becomes `prepareforsaveiv`), so a plain substring
+ * test reads short algorithm names inside unrelated identifiers (`rsa` in
+ * `for_save`). A name therefore matches only where a cleanStr word begins:
+ * the snippet's start, right after any character cleanStr strips or keeps as
+ * a separator, or at a lowercase-to-uppercase change in the original text
+ * (`SHA256withRSA`, `HmacSHA256`, `PBKDF2WithHmacSHA256`,
+ * `generateRsaKeyPair`). Boundaries are computed from the original string
+ * before it is lowercased, so case changes survive the cleanStr fusion.
+ * Letter/digit changes do not start a word: they added no match on any
+ * reviewed form or corpus snippet, while splitting every digit-bearing
+ * identifier (`md5`, `base64`) into false word starts. Fused forms the names
+ * themselves are written in still match: `algorithms.AES(key)` and
+ * `AESGCM(key)` yield `aes`, `AES-256-GCM` yields `aes256-GCM`,
+ * `createHash("sha256")` yields `sha-256`. All-uppercase fused forms
+ * (`HMACSHA256`) still hide every name but the run's first.
+ *
  * @param {string} code Code snippet
  * @returns {Array} Arary of crypto algorithm objects with oid and description
  */
 export function findCryptoAlgos(code) {
-  const cleanCode = cleanStr(code);
+  if (typeof code !== "string" || !code) {
+    return [];
+  }
+  const cleanChars = [];
+  // canStart[i] is true when cleanChars[i] begins a maximal [0-9a-z] word,
+  // either in the original text or at a lower-to-uppercase change in it
+  const canStart = [];
+  let inWord = false;
+  for (let i = 0; i < code.length; i++) {
+    const ch = code[i];
+    const lower = ch.toLowerCase();
+    const isWordChar =
+      lower.length === 1 &&
+      ((lower >= "0" && lower <= "9") || (lower >= "a" && lower <= "z"));
+    if (isWordChar) {
+      cleanChars.push(lower);
+      const prev = i > 0 ? code[i - 1] : "";
+      const caseStart = /[a-z]/.test(prev) && /[A-Z]/.test(ch);
+      canStart.push(!inWord || caseStart);
+      inWord = true;
+    } else {
+      inWord = false;
+    }
+  }
+  const cleanCode = cleanChars.join("");
   const cryptoAlgos = [];
   for (const algoName of Object.keys(cbomCryptoOids)) {
-    if (cleanCode.includes(cleanStr(algoName))) {
-      cryptoAlgos.push({
-        ...cbomCryptoOids[algoName],
-        name: algoName,
-        ref: `crypto/algorithm/${algoName}@${cbomCryptoOids[algoName].oid}`,
-      });
+    const needle = cleanStr(algoName);
+    if (!needle) {
+      continue;
+    }
+    let pos = cleanCode.indexOf(needle);
+    for (; pos !== -1; pos = cleanCode.indexOf(needle, pos + 1)) {
+      if (canStart[pos]) {
+        cryptoAlgos.push({
+          ...cbomCryptoOids[algoName],
+          name: algoName,
+          ref: `crypto/algorithm/${algoName}@${cbomCryptoOids[algoName].oid}`,
+        });
+        break;
+      }
     }
   }
   return cryptoAlgos;

package/lib/inventory/deps.js

@@ -25,6 +25,10 @@ import {
   mavenLocalRepositories,
   splitArtifactFileName,
 } from "./jvmLocalRepos.js";
+import {
+  mavenMirrorSettingsArgs,
+  mavenWrapperMirrorEnv,
+} from "./mavenCentralMirror.js";
 import { pypiBomRef } from "./purl.js";
 import { findLicenseId, spdxLicenses } from "./spdx.js";
 
@@ -136,11 +140,13 @@ export async function collectMvnDependencies(
     const addArgs = parseMavenArgs(readEnvironmentVariable("MVN_ARGS"));
     copyArgs = copyArgs.concat(addArgs);
   }
+  copyArgs = copyArgs.concat(mavenMirrorSettingsArgs());
   if (basePath && basePath !== MAVEN_CACHE_DIR) {
     console.log(`Executing '${mavenCmd} in ${basePath}`);
     const result = safeSpawnSync(mavenCmd, copyArgs, {
       cwd: basePath,
       shell: isWin,
+      env: { ...process.env, ...mavenWrapperMirrorEnv() },
     });
     const rateLimited = noteBuildToolRateLimit(
       "maven",
@@ -336,6 +342,35 @@ export async function collectJarNS(jarPath, pomPathMap = {}) {
   return jarNSMapping;
 }
 
+/**
+ * The jar namespace map entries of components a jar scan produced, in the
+ * shape {@link collectJarNS} writes (`{ namespaces: string[] }`), from their
+ * `internal:Namespaces` properties. The scan reads each jar from a temporary
+ * copy, so no `jarFile` is recorded. Purls `existing` already maps keep the
+ * entry they have, which names the jar it was read from.
+ *
+ * @param {Object[]} components Components with `internal:Namespaces` properties
+ * @param {Object} [existing] Jar namespace map already collected
+ * @returns {Object} New entries keyed by purl
+ */
+export function namespaceMappingOfComponents(components, existing = {}) {
+  const mapping = {};
+  for (const component of components || []) {
+    if (!component?.purl || existing[component.purl]) {
+      continue;
+    }
+    const namespaces = (component.properties || [])
+      .filter((property) => property.name === "internal:Namespaces")
+      .flatMap((property) => String(property.value || "").split("\n"))
+      .map((namespace) => namespace.trim())
+      .filter(Boolean);
+    if (namespaces.length) {
+      mapping[component.purl] = { namespaces: [...new Set(namespaces)] };
+    }
+  }
+  return mapping;
+}
+
 /**
  * Convert a JAR namespace mapping (produced by {@link collectJarNS}) into an array
  * of CycloneDX package component objects.

package/lib/inventory/dosai.js

@@ -727,6 +727,105 @@ function referencedIds(entries, field) {
   return ids;
 }
 
+// The fields of a MethodCalls row cdxgen reads: the component it names (Purl
+// or Module), where the call site is (Path, FileName, LineNumber) and what it
+// calls (ClassName, CalledMethod). Everything else a report row carries -
+// arguments, evidence kinds, caller identities - no cdxgen consumer reads,
+// and a report with millions of calls cannot afford to hold them.
+const DOSAI_METHOD_CALL_FIELDS = [
+  "Purl",
+  "Module",
+  "Path",
+  "FileName",
+  "LineNumber",
+  "ClassName",
+  "CalledMethod",
+];
+
+/**
+ * The largest number of distinct method calls cdxgen keeps from one report.
+ *
+ * Reports in the millions of calls repeat the same call sites heavily, so the
+ * aggregated rows stay far below this; the cap exists so a report whose calls
+ * are all distinct degrades the occurrence evidence instead of the process.
+ *
+ * @returns {number} Cap on distinct kept rows
+ */
+export function maxDosaiMethodCalls() {
+  const configured = Number.parseInt(
+    readEnvironmentVariable("CDXGEN_DOSAI_MAX_METHOD_CALLS") || "",
+    10,
+  );
+  if (Number.isInteger(configured) && configured > 0) {
+    return configured;
+  }
+  return 1_000_000;
+}
+
+/**
+ * Build the `map` that turns report MethodCalls rows into what cdxgen keeps:
+ * rows the caller rejects are dropped, the rest are reduced to
+ * {@link DOSAI_METHOD_CALL_FIELDS} and aggregated by those fields, so a call
+ * site reached a million times is held once with a `CallCount`. Past
+ * {@link maxDosaiMethodCalls} distinct rows, further rows are dropped and one
+ * warning names the cap.
+ *
+ * @param {string} reportFile Report the rows come from, named in the warning
+ * @param {Object} [options] Options
+ * @param {Function} [options.keepMethodCall] Predicate choosing the rows to keep
+ * @param {number} [options.maxMethodCalls] Cap override, for tests
+ * @returns {{map: Function, finish: Function}} The row mapper, and the
+ *   `finish` that reports a cap reached
+ */
+function dosaiMethodCallCollector(reportFile, options = {}) {
+  const keep = options.keepMethodCall;
+  const max = options.maxMethodCalls ?? maxDosaiMethodCalls();
+  const keptByKey = new Map();
+  let dropped = 0;
+  return {
+    map(methodCall) {
+      if (keep && !keep(methodCall)) {
+        return undefined;
+      }
+      const compact = {};
+      for (const field of DOSAI_METHOD_CALL_FIELDS) {
+        if (methodCall?.[field] !== undefined) {
+          compact[field] = methodCall[field];
+        }
+      }
+      const key = DOSAI_METHOD_CALL_FIELDS.map((field) => compact[field]).join(
+        "\u0000",
+      );
+      const kept = keptByKey.get(key);
+      if (kept) {
+        kept.CallCount = (kept.CallCount ?? 1) + 1;
+        return undefined;
+      }
+      if (keptByKey.size >= max) {
+        dropped++;
+        return undefined;
+      }
+      keptByKey.set(key, compact);
+      return compact;
+    },
+    finish() {
+      if (!dropped) {
+        return;
+      }
+      console.warn(
+        `The dosai report "${reportFile}" holds more distinct method calls than cdxgen keeps (${max}); ${dropped} call sites were dropped, so occurrence evidence may be incomplete. Raise the cap with CDXGEN_DOSAI_MAX_METHOD_CALLS.`,
+      );
+      recordDegradation("dosai.method-calls-capped", {
+        ecosystem: "csharp",
+        tool: "dosai",
+        impact: "components",
+        command: reportFile,
+        detail: `The dosai report holds more distinct method calls than the cap of ${max}, so ${dropped} call sites were dropped from the occurrence evidence.`,
+      });
+    },
+  };
+}
+
 function readDosaiReport(reportFile, options, readTrimmed) {
   if (!reportFile || !safeExistsSync(reportFile)) {
     return undefined;
@@ -761,7 +860,13 @@ function readDosaiReport(reportFile, options, readTrimmed) {
     }
     return report;
   } catch (err) {
-    if (DEBUG_MODE) {
+    if (err instanceof RangeError) {
+      // A heap or string limit is a report-size problem the user can act on,
+      // unlike a malformed report, so it is named even without debug output.
+      console.warn(
+        `The dosai report "${reportFile}" is too large to read: ${err.message}. Raise the Node.js heap, for example NODE_OPTIONS="--max-old-space-size=8192", or write the report with dosai's --sections so it holds less.`,
+      );
+    } else if (DEBUG_MODE) {
       console.log(
         `Unable to read the dosai report "${reportFile}": ${err.message}`,
       );
@@ -776,25 +881,34 @@ function readDosaiReport(reportFile, options, readTrimmed) {
  * A report that fits in one JavaScript string is parsed whole. A larger one is
  * read in bounded runs, keeping the requested sections, the CallGraph nodes and
  * edges that PackageReachability references, and the MethodCalls accepted by
- * `keepMethodCall` (none when it is omitted). Nothing cdxgen reads is lost.
+ * `keepMethodCall` (none when it is omitted). A kept MethodCalls row is
+ * reduced to the fields cdxgen reads and aggregated by them, so a call site
+ * reached many times is held once with a `CallCount`; distinct rows past
+ * {@link maxDosaiMethodCalls} are dropped with one warning. Nothing cdxgen
+ * reads is lost.
  *
  * @param {string} reportFile Path to the dosai methods JSON
  * @param {Object} [options] Options
  * @param {string[]} [options.sections] Sections to keep from a large report (default: every section cdxgen reads)
  * @param {Function} [options.keepMethodCall] Predicate choosing the MethodCalls to keep from a large report
+ * @param {number} [options.maxMethodCalls] Cap on distinct kept MethodCalls, for tests
  * @param {number} [options.maxTextBytes] Largest report parsed whole, for tests
  * @returns {Object|undefined} Parsed report, or undefined when missing or invalid
  */
 export function readDosaiMethodsReport(reportFile, options = {}) {
   const sections = options.sections || DOSAI_METHODS_SECTIONS;
-  return readDosaiReport(reportFile, options, (report, read) => {
+  const collector =
+    sections.includes("MethodCalls") && options.keepMethodCall
+      ? dosaiMethodCallCollector(reportFile, options)
+      : undefined;
+  const report = readDosaiReport(reportFile, options, (report, read) => {
     for (const section of sections) {
       if (section !== "CallGraph" && section !== "MethodCalls") {
         read(section);
       }
     }
-    if (sections.includes("MethodCalls") && options.keepMethodCall) {
-      read("MethodCalls", { filter: options.keepMethodCall });
+    if (collector) {
+      read("MethodCalls", { map: collector.map });
     }
     if (sections.includes("CallGraph")) {
       // CallGraph precedes PackageReachability in the file, which is why the
@@ -809,6 +923,26 @@ export function readDosaiMethodsReport(reportFile, options = {}) {
       });
     }
   });
+  // A report parsed whole kept every row, as it always has: reduce them to
+  // the fields cdxgen reads and aggregate repeats, without the caller's
+  // predicate, which the whole parse has never applied. A report that took
+  // the trimmed path already went through the collector above.
+  if (report && !dosaiReportSources.get(report)?.trimmed) {
+    const wholeCollector = dosaiMethodCallCollector(reportFile, {
+      maxMethodCalls: options.maxMethodCalls,
+    });
+    const reduced = [];
+    for (const methodCall of report.MethodCalls || []) {
+      const kept = wholeCollector.map(methodCall);
+      if (kept !== undefined) {
+        reduced.push(kept);
+      }
+    }
+    report.MethodCalls = reduced;
+    wholeCollector.finish();
+  }
+  collector?.finish();
+  return report;
 }
 
 /**