@cdxgen/cdxgen 13.5.0 → 13.7.0

package/lib/inventory/scalaDedupe.js

@@ -0,0 +1,440 @@
+import { basename } from "node:path";
+
+import { Purl } from "@cdxgen/cdx-purl";
+
+import { mergeDependencies, trimComponents } from "./depsUtils.js";
+import { mavenPurl } from "./purl.js";
+import { publishedArtifactId, scalaCoordinateOfPurl } from "./scalaCoords.js";
+
+const COMPILER_VERSION_PROPERTY = "cdx:scala:compilerVersion";
+
+// A jar scan that read the coordinates from the jar's own pom.properties
+// records them with this confidence; a guess from the file name or the
+// manifest records less.
+const DESCRIPTOR_CONFIDENCE = 0.5;
+
+function compilerVersionsOf(component) {
+  return (component.properties || [])
+    .filter((property) => property.name === COMPILER_VERSION_PROPERTY)
+    .map((property) => property.value)
+    .filter(Boolean);
+}
+
+function parsedPurl(component) {
+  try {
+    return Purl.parse(component?.purl || "");
+  } catch (_err) {
+    return undefined;
+  }
+}
+
+// The jar names a component was read from, without the extension.
+function jarStems(component) {
+  return (component.properties || [])
+    .filter((property) => property.name === "internal:SrcFile")
+    .map((property) => basename(String(property.value || "")))
+    .filter((name) => name.toLowerCase().endsWith(".jar"))
+    .map((name) => name.slice(0, -".jar".length));
+}
+
+function isJarSourced(component) {
+  return jarStems(component).length > 0;
+}
+
+// Whether a jar scan guessed the coordinates rather than reading them from the
+// jar's Maven descriptor. A jar with a descriptor names its own artifact, even
+// when that artifact is spelled `<group>.<name>` like a staged copy.
+function hasGuessedIdentity(component) {
+  const identities = [component.evidence?.identity || []].flat();
+  return !identities.some(
+    (identity) =>
+      (identity?.confidence ?? 0) >= DESCRIPTOR_CONFIDENCE ||
+      (identity?.methods || []).some(
+        (method) => (method?.confidence ?? 0) >= DESCRIPTOR_CONFIDENCE,
+      ),
+  );
+}
+
+// A classifier or a type other than jar names a different artifact of the same
+// coordinates, which is not a duplicate.
+function isPlainJar(component) {
+  const purl = parsedPurl(component);
+  if (!purl) {
+    return false;
+  }
+  const qualifiers = purl.qualifiers || {};
+  return (
+    !qualifiers.classifier && (!qualifiers.type || qualifiers.type === "jar")
+  );
+}
+
+// The Scala binary versions a component was built for: the suffix its name
+// keeps, or the compiler versions a build tool recorded. Empty for a library
+// that is not a Scala artifact.
+function binaryVersionsOf(component) {
+  const suffixVersion = scalaCoordinateOfPurl(component.purl)?.binaryVersion;
+  return new Set(
+    suffixVersion ? [suffixVersion] : compilerVersionsOf(component),
+  );
+}
+
+// Two spellings name one artifact only when they were built for the same Scala
+// binary version, or both for none. `kafka_2.12` and `kafka_2.13` share the
+// normalized coordinates but are different artifacts.
+function sameBinaryVersion(a, b) {
+  const left = binaryVersionsOf(a);
+  const right = binaryVersionsOf(b);
+  if (!left.size || !right.size) {
+    return !left.size && !right.size;
+  }
+  return [...left].some((version) => right.has(version));
+}
+
+// Lower is better. A component a build tool reported outranks one a jar scan
+// named; then the name without a `<group>.` prefix, then the purl that names
+// its repository, then the shorter purl, then the lexicographically first, so
+// the choice never depends on the order of the BOM.
+function rank(component) {
+  return [
+    isJarSourced(component) ? 1 : 0,
+    component.name?.startsWith(`${component.group}.`) ? 1 : 0,
+    component.purl.includes("repository_url=") ? 0 : 1,
+    component.purl.length,
+    component.purl,
+  ];
+}
+
+function isBetter(candidate, component) {
+  const a = rank(candidate);
+  const b = rank(component);
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) {
+      return a[i] < b[i];
+    }
+  }
+  return false;
+}
+
+function best(components) {
+  let chosen;
+  for (const component of components) {
+    if (!chosen || isBetter(component, chosen)) {
+      chosen = component;
+    }
+  }
+  return chosen;
+}
+
+function refOf(component) {
+  return component["bom-ref"] || component.purl;
+}
+
+// The file names sbt-native-packager gives a staged copy of a library,
+// without the version: `<group>.<artifact>` with the artifact as published,
+// once for each Scala binary version it was built for, and once as named,
+// since a module of the build itself is staged without the suffix when it
+// turns cross paths off.
+function stagedStems(component) {
+  return [
+    ...new Set([
+      `${component.group}.${component.name}`,
+      ...compilerVersionsOf(component).map(
+        (version) =>
+          `${component.group}.${publishedArtifactId({
+            name: component.name,
+            properties: [{ name: COMPILER_VERSION_PROPERTY, value: version }],
+          })}`,
+      ),
+    ]),
+  ];
+}
+
+// The listed libraries a staged file name can be a copy of, with the version
+// the name carries: every split of the name at a hyphen whose left side is the
+// staged stem of a library.
+function stagedOwners(stem, ownersByStem) {
+  const found = [];
+  for (
+    let index = stem.indexOf("-");
+    index > 0;
+    index = stem.indexOf("-", index + 1)
+  ) {
+    const owners = ownersByStem.get(stem.slice(0, index));
+    if (owners) {
+      found.push({ owners, version: stem.slice(index + 1) });
+    }
+  }
+  return found;
+}
+
+function rewriteReferences(dependencies, refMap) {
+  if (!refMap.size) {
+    return dependencies;
+  }
+  const rewritten = (dependencies || []).map((dependency) => {
+    const ref = refMap.get(dependency.ref) || dependency.ref;
+    return {
+      ...dependency,
+      ref,
+      ...(dependency.dependsOn
+        ? {
+            dependsOn: [
+              ...new Set(
+                dependency.dependsOn.map(
+                  (dependsOn) => refMap.get(dependsOn) || dependsOn,
+                ),
+              ),
+            ].filter((dependsOn) => dependsOn !== ref),
+          }
+        : {}),
+    };
+  });
+  return mergeDependencies([], rewritten);
+}
+
+/**
+ * Collapse the components that stand for one Scala library.
+ *
+ * A build tool such as sbt reports a library as `group:name@version` with the
+ * Scala binary suffix moved into `cdx:scala:compilerVersion`. The jars a scan
+ * finds on disk name the same library again, and not in the same way: with the
+ * suffix (`cats-core_3`), without a repository, or, for a jar without a Maven
+ * descriptor, only as the staged file name `<group>.<artifact>-<version>.jar`,
+ * which a filename-based guess splits at the wrong dot
+ * (`pkg:maven/cpg2-protos/io.appthreat.cpg2-protos_3`). Each of those is merged
+ * into the component the build tool reported. A staged jar of another version
+ * of a reported library, such as a stale build, keeps its own component but is
+ * renamed to the coordinates the reported library has.
+ *
+ * Spellings of the same coordinates merge only when they were built for the
+ * same Scala binary version, so the `_2.12` and `_2.13` artifacts of one
+ * library stay two components. Components that no reported library accounts
+ * for are left as they are.
+ *
+ * @param {Object[]} components BOM components
+ * @param {Object[]} dependencies BOM dependencies
+ * @returns {{components: Object[], dependencies: Object[], purlMap: Map<string, string>}}
+ *   The collapsed components, the dependencies with references to merged
+ *   components pointed at the component that remains, and the purl each
+ *   merged or renamed component now has, keyed by the purl it had
+ */
+export function mergeScalaDuplicateComponents(components, dependencies) {
+  const purlMap = new Map();
+  const candidates = (components || []).filter(
+    (component) =>
+      component?.purl?.startsWith("pkg:maven/") &&
+      isPlainJar(component) &&
+      component.group &&
+      component.version,
+  );
+  if (candidates.length < 2) {
+    return { components, dependencies, purlMap };
+  }
+  const byKey = new Map();
+  const ownersByStem = new Map();
+  for (const component of candidates) {
+    const key = scalaCoordinateOfPurl(component.purl)?.key;
+    if (key) {
+      byKey.set(key, [...(byKey.get(key) || []), component]);
+    }
+    for (const stem of stagedStems(component)) {
+      ownersByStem.set(stem, [...(ownersByStem.get(stem) || []), component]);
+    }
+  }
+  // Which component each one is a duplicate of.
+  const duplicateOf = new Map();
+  const repoints = new Map();
+  for (const component of candidates) {
+    const matches = new Set();
+    // The same coordinates, spelled with or without the Scala suffix, with or
+    // without a repository.
+    for (const other of byKey.get(scalaCoordinateOfPurl(component.purl)?.key) ||
+      []) {
+      if (other !== component && sameBinaryVersion(component, other)) {
+        matches.add(other);
+      }
+    }
+    // The staged file name of another listed library, whatever group and name
+    // the guess made of it. A jar that names itself in its Maven descriptor is
+    // the artifact it says it is.
+    if (isJarSourced(component) && hasGuessedIdentity(component)) {
+      for (const stem of jarStems(component)) {
+        for (const { owners, version } of stagedOwners(stem, ownersByStem)) {
+          const sameVersion = owners.filter(
+            (owner) => owner !== component && owner.version === version,
+          );
+          if (sameVersion.length) {
+            for (const owner of sameVersion) {
+              matches.add(owner);
+            }
+            continue;
+          }
+          // Another version of a library a build tool reported, when the guess
+          // agrees on it. A version that extends a known one
+          // (`4.1.100.Final-linux-x86_64`) can as well be that version with a
+          // classifier, so it is left alone.
+          const reportedOwners = owners.filter(
+            (owner) => owner !== component && !isJarSourced(owner),
+          );
+          const extendsKnown = owners.some((owner) =>
+            version.startsWith(`${owner.version}-`),
+          );
+          if (
+            reportedOwners.length &&
+            version === component.version &&
+            /^\d/.test(version) &&
+            !extendsKnown
+          ) {
+            const owner = best(reportedOwners);
+            const known = repoints.get(component);
+            if (!known || isBetter(owner, known)) {
+              repoints.set(component, owner);
+            }
+          }
+        }
+      }
+    }
+    const target = best(
+      [...matches].filter((other) => isBetter(other, component)),
+    );
+    if (target) {
+      duplicateOf.set(component, target);
+    }
+  }
+  // A chain of merges never carries a component past one built for another
+  // Scala binary version.
+  const root = (component) => {
+    let current = component;
+    while (duplicateOf.has(current)) {
+      const next = duplicateOf.get(current);
+      if (current !== component && !sameBinaryVersion(component, next)) {
+        break;
+      }
+      current = next;
+    }
+    return current;
+  };
+  const refMap = new Map();
+  const survivors = new Set(components);
+  for (const [component] of duplicateOf) {
+    const target = root(component);
+    const duplicate = { ...component };
+    const targetRef = refOf(target);
+    if (refOf(component) && targetRef && refOf(component) !== targetRef) {
+      refMap.set(refOf(component), targetRef);
+    }
+    if (component.purl !== target.purl) {
+      purlMap.set(component.purl, target.purl);
+    }
+    duplicate.purl = target.purl;
+    duplicate["bom-ref"] = target["bom-ref"];
+    trimComponents([target, duplicate]);
+    survivors.delete(component);
+  }
+  for (const [component, owner] of repoints) {
+    if (survivors.has(component)) {
+      repointToAnchor(component, root(owner), refMap, purlMap);
+    }
+  }
+  // A component merged into one that was renamed afterwards follows it.
+  resolveChains(refMap);
+  resolveChains(purlMap);
+  // Repointed components can now spell the same coordinates twice.
+  return {
+    components: trimComponents([...survivors]),
+    dependencies: rewriteReferences(dependencies, refMap),
+    purlMap,
+  };
+}
+
+function resolveChains(map) {
+  for (const [key, value] of map) {
+    let current = value;
+    const seen = new Set([key]);
+    while (map.has(current) && !seen.has(current)) {
+      seen.add(current);
+      current = map.get(current);
+    }
+    map.set(key, current);
+  }
+}
+
+// Give a staged jar of another version the group, name and purl shape the
+// reported library has, so it is the same library at an older or newer version
+// and not a differently named one.
+function repointToAnchor(component, anchor, refMap, purlMap) {
+  const anchorPurl = parsedPurl(anchor);
+  if (!anchorPurl) {
+    return;
+  }
+  // The repository the reported version came from says nothing about where
+  // this one came from.
+  const { repository_url: _repository, ...qualifiers } =
+    anchorPurl.qualifiers || {};
+  const purl = mavenPurl(
+    anchor.group,
+    anchorPurl.name,
+    component.version,
+    qualifiers,
+  );
+  if (!purl) {
+    return;
+  }
+  const previousRef = refOf(component);
+  purlMap.set(component.purl, purl);
+  component.group = anchor.group;
+  component.name = anchor.name;
+  component.purl = purl;
+  component["bom-ref"] = decodeURIComponent(purl);
+  const known = new Set(compilerVersionsOf(component));
+  component.properties = [
+    ...(component.properties || []),
+    ...(anchor.properties || []).filter(
+      (property) =>
+        property.name === COMPILER_VERSION_PROPERTY &&
+        !known.has(property.value),
+    ),
+  ];
+  if (previousRef && previousRef !== component["bom-ref"]) {
+    refMap.set(previousRef, component["bom-ref"]);
+  }
+}
+
+/**
+ * Key the jar namespace map by the purls the components have after
+ * {@link mergeScalaDuplicateComponents}, so a merged or renamed jar still
+ * finds its namespaces. A purl that already has an entry keeps it, with the
+ * namespaces of the merged entry added.
+ *
+ * @param {Object} mapping Jar namespace map keyed by purl
+ * @param {Map<string, string>} purlMap New purl keyed by old purl
+ * @returns {Object} The re-keyed map
+ */
+export function rekeyNamespaceMapping(mapping, purlMap) {
+  if (!mapping || !purlMap?.size) {
+    return mapping;
+  }
+  const rekeyed = { ...mapping };
+  for (const [oldPurl, newPurl] of purlMap) {
+    const entry = rekeyed[oldPurl];
+    if (!entry || oldPurl === newPurl) {
+      continue;
+    }
+    delete rekeyed[oldPurl];
+    const existing = rekeyed[newPurl];
+    if (!existing) {
+      rekeyed[newPurl] = entry;
+      continue;
+    }
+    rekeyed[newPurl] = {
+      ...existing,
+      namespaces: [
+        ...new Set([
+          ...(existing.namespaces || []),
+          ...(entry.namespaces || []),
+        ]),
+      ],
+    };
+  }
+  return rekeyed;
+}

package/lib/inventory/source.js

@@ -30,6 +30,7 @@ import {
   prefetchedResponse,
   prefetchJson,
 } from "./fetchBatch.js";
+import { mavenCentralFetchUrl } from "./mavenCentralMirror.js";
 import { canonicalPurlFromLooseString } from "./purl.js";
 
 /** Raised when a caller omits a registry helper that the purl type requires. */
@@ -105,9 +106,48 @@ function inferGitOperation(args = []) {
   return "command";
 }
 
+let gitRedirectRefusalReported = false;
+
+registerRunReset("source:git-redirect-refusal", () => {
+  gitRedirectRefusalReported = false;
+});
+
+/**
+ * Determine whether git must refuse HTTP redirects. A git host allowlist is
+ * checked against the URL cdxgen is given, and git would otherwise follow
+ * the server's redirect on its first request to a host the allowlist never
+ * saw. Secure mode refuses redirects in the HTTP client too.
+ *
+ * @returns {boolean} true when a git host allowlist or secure mode applies
+ */
+export function gitRedirectsRefused() {
+  return isSecureMode || getAllowedHosts().length > 0;
+}
+
+/**
+ * Determine whether a failed git command stopped at an HTTP redirect.
+ *
+ * @param {string|undefined} stderr git error output
+ * @returns {boolean} true when git reports a 3xx response
+ */
+function isGitRedirectRefusal(stderr) {
+  if (typeof stderr !== "string") {
+    return false;
+  }
+  const marker = "The requested URL returned error: 30";
+  const index = stderr.indexOf(marker);
+  return index !== -1 && "12378".includes(stderr[index + marker.length]);
+}
+
 /**
  * Execute git with hardened defaults.
  *
+ * Every call runs with command-scope settings that keep the repository from
+ * running fsmonitor hooks or being opened as a bare repository. While a git
+ * host allowlist or secure mode applies, git also refuses HTTP redirects, so
+ * an allowed host cannot send a clone or ls-remote to a host the allowlist
+ * does not name.
+ *
  * @param {string[]} args git arguments
  * @param {Object} options command options
  * @param {string|undefined} options.cwd working directory
@@ -119,15 +159,23 @@ function inferGitOperation(args = []) {
 export function hardenedGitCommand(args, options = {}) {
   const gitOperation = inferGitOperation(args);
   const gitAllowProtocol = getGitAllowProtocol();
+  const refuseRedirects = gitRedirectsRefused();
+  const commandConfig = [
+    ["core.fsmonitor", "false"],
+    ["safe.bareRepository", "explicit"],
+  ];
+  if (refuseRedirects) {
+    commandConfig.push(["http.followRedirects", "false"]);
+  }
   const envConfigs = {
-    GIT_CONFIG_COUNT: "2",
-    GIT_CONFIG_KEY_0: "core.fsmonitor",
-    GIT_CONFIG_VALUE_0: "false",
-    GIT_CONFIG_KEY_1: "safe.bareRepository",
-    GIT_CONFIG_VALUE_1: "explicit",
+    GIT_CONFIG_COUNT: String(commandConfig.length),
     GIT_NO_LAZY_FETCH: "1",
     GIT_TERMINAL_PROMPT: "0",
   };
+  commandConfig.forEach(([key, value], index) => {
+    envConfigs[`GIT_CONFIG_KEY_${index}`] = key;
+    envConfigs[`GIT_CONFIG_VALUE_${index}`] = value;
+  });
   if (options.noRepository) {
     // ls-remote works without a repository. Pointing GIT_DIR at a path that
     // does not exist stops git from discovering one in the working directory.
@@ -146,7 +194,7 @@ export function hardenedGitCommand(args, options = {}) {
         ...envConfigs,
         GIT_ALLOW_PROTOCOL: gitAllowProtocol,
       };
-  return safeSpawnSync("git", args, {
+  const result = safeSpawnSync("git", args, {
     cdxgenActivity: {
       blockedReason: `Dry run mode blocks git ${gitOperation} operations.`,
       gitOperation,
@@ -159,6 +207,18 @@ export function hardenedGitCommand(args, options = {}) {
     cwd: options.cwd,
     env,
   });
+  if (
+    refuseRedirects &&
+    !gitRedirectRefusalReported &&
+    result?.status !== 0 &&
+    isGitRedirectRefusal(result?.stderr)
+  ) {
+    gitRedirectRefusalReported = true;
+    console.warn(
+      "A Git server answered with a redirect, which cdxgen does not follow while CDXGEN_GIT_ALLOWED_HOSTS, CDXGEN_SERVER_ALLOWED_HOSTS or secure mode is in effect. Use the repository's current URL instead.",
+    );
+  }
+  return result;
 }
 
 function normalizeTagName(tagName) {
@@ -795,6 +855,48 @@ export function getGitRemoteHost(remote) {
   return host.toLowerCase();
 }
 
+/**
+ * Remove credentials from git output before it is logged. Every URL in the
+ * text loses its user name and password, which git copies into its error
+ * messages from the remote it was given.
+ *
+ * @param {string|undefined} text Output of a git command
+ * @param {string|undefined} repoUrl Remote the command was given
+ * @returns {string|undefined} Output with credentials replaced by ***
+ */
+export function redactGitOutput(text, repoUrl) {
+  if (!text || typeof text !== "string") {
+    return text;
+  }
+  let redacted = text;
+  if (repoUrl && typeof repoUrl === "string") {
+    redacted = redacted.split(repoUrl).join(sanitizeRemoteUrlForLogs(repoUrl));
+  }
+  let from = 0;
+  while (from < redacted.length) {
+    const schemeEnd = redacted.indexOf("://", from);
+    if (schemeEnd === -1) {
+      break;
+    }
+    const authorityStart = schemeEnd + 3;
+    let authorityEnd = authorityStart;
+    while (
+      authorityEnd < redacted.length &&
+      !"/?#'\"<> \t\r\n".includes(redacted[authorityEnd])
+    ) {
+      authorityEnd++;
+    }
+    const at = redacted.lastIndexOf("@", authorityEnd - 1);
+    if (at >= authorityStart) {
+      redacted = `${redacted.slice(0, authorityStart)}***:***${redacted.slice(at)}`;
+      from = authorityStart + "***:***".length;
+    } else {
+      from = authorityEnd;
+    }
+  }
+  return redacted;
+}
+
 /**
  * Clone a git repository into a temporary directory.
  *
@@ -859,8 +961,11 @@ export function gitClone(repoUrl, branch = null) {
     `Cloning Repo${branch ? ` with branch ${branch}` : ""} to ${tempDir}`,
   );
   const result = hardenedGitCommand(gitArgs);
-  if (result.status !== 0) {
-    console.error(result.stderr);
+  if (result.status !== 0 || result.error) {
+    console.error(redactGitOutput(result.stderr, repoUrl));
+    // The directory was created by this call, so it is always safe to remove.
+    safeRmSync(tempDir, { recursive: true, force: true });
+    throw new Error("Git clone failed.");
   }
 
   return tempDir;
@@ -1588,6 +1693,7 @@ export async function resolveGitUrlFromPurl(purlString, helpers = {}) {
           break;
         }
         const mavenCentralUrl =
+          mavenCentralFetchUrl() ||
           readEnvironmentVariable("MAVEN_CENTRAL_URL") ||
           "https://repo1.maven.org/maven2/";
         registry = mavenCentralUrl.endsWith("/")