@cdxgen/cdxgen 13.0.1 → 13.2.0

package/lib/ecosystems/ecosystems.js

@@ -66,11 +66,13 @@ import {
   prefetchEnabled,
   prefetchedResponse,
   prefetchJson,
+  withHostRateLimit,
 } from "../inventory/fetchBatch.js";
 import {
   applyPurl,
   encodeForPurl,
   genericPurl,
+  nugetPurl,
   pypiPurl,
 } from "../inventory/purl.js";
 import {
@@ -80,6 +82,14 @@ import {
 } from "../inventory/spdx.js";
 import { extractLicenseText, extractRepoUrl } from "../parsers/htmlExtract.js";
 import { xml2js } from "../parsers/xml.js";
+import {
+  localCargoMetadataEnabled,
+  readCargoCacheMetadata,
+} from "./cargoCache.js";
+import {
+  applyPypiClassifierMetadata,
+  applyPypiModuleNames,
+} from "./pypiClassifiers.js";
 import {
   collectCargoRegistryProvenanceProperties,
   collectNpmRegistryProvenanceProperties,
@@ -810,11 +820,13 @@ export async function fetchPomXml({ urlPrefix, group, name, version }) {
   // A direct POM may have been prefetched by getMvnMetadata's batch round.
   // Parent POMs (fetched from inside fetchPomXmlAsJson) are not prefetched
   // because their URL depends on the direct POM's content, so they fall
-  // through to the serial request unchanged.
+  // through to the serial request. That fallback still reaches Maven Central
+  // and still counts against the same budget as the batch round, so it queues
+  // behind the same per-host gate rather than a second, ungoverned one.
   try {
     const res =
       prefetchedResponse(prefetchedPoms, fullUrl) ||
-      (await cdxgenAgent.get(fullUrl));
+      (await withHostRateLimit(fullUrl, () => cdxgenAgent.get(fullUrl)));
     return res.body;
   } catch (_err) {
     return undefined;
@@ -950,6 +962,10 @@ export async function getPyMetadata(pkgList, fetchDepsInfo) {
       continue;
     }
     try {
+      // Module names are derived from the distribution name and the curated alias
+      // map, so they are available without a registry round trip and are attached
+      // before the skip below rather than after it.
+      applyPypiModuleNames(p);
       // If the package name has a url or already includes license and version skip it
       if (p.name.includes("https") || (p.license && p.version)) {
         cdepList.push(p);
@@ -1011,6 +1027,7 @@ export async function getPyMetadata(pkgList, fetchDepsInfo) {
             }
           }
         }
+        applyPypiClassifierMetadata(p, body.info.classifiers);
       }
       if (body.info.license) {
         const licenseId = findLicenseId(body.info.license);
@@ -1783,24 +1800,61 @@ function getGoPkgFullName(group, name) {
 /**
  * Method to retrieve metadata for rust packages by querying crates
  *
+ * The local Cargo registry is consulted first, because crates.io's crawler
+ * policy allows one request per second and a populated `~/.cargo` already
+ * holds the license, description, repository, checksum and yanked flag for
+ * every crate the build compiled.
+ *
+ * What it does not hold is publisher identity, so a caller that needs the
+ * publisher-drift and release-cadence signals — the predictive audit does —
+ * passes `preferLocalCache: false` and takes the slower registry path.
+ *
  * @param {Array} pkgList Package list
+ * @param {Object} [options] Options
+ * @param {boolean} [options.preferLocalCache=true] Answer from the local Cargo
+ *   registry where it can, and query crates.io only for the rest.
  */
-export async function getCratesMetadata(pkgList) {
+export async function getCratesMetadata(pkgList, options = {}) {
   const CRATES_URL =
     readEnvironmentVariable("RUST_CRATES_URL") ||
     "https://crates.io/api/v1/crates/";
   const cdepList = [];
+  // The local Cargo registry answers first. crates.io allows one request per
+  // second, so a populated ~/.cargo turns minutes of waiting into a few file
+  // reads; only the crates it cannot account for reach the network.
+  const servedLocally = new Set();
+  const preferLocalCache = options.preferLocalCache !== false;
+  if (preferLocalCache && localCargoMetadataEnabled()) {
+    for (const p of pkgList) {
+      if (!p?.name || !p?.version || p.version === "workspace") {
+        continue;
+      }
+      if (applyCargoCacheMetadata(p)) {
+        servedLocally.add(p);
+      }
+    }
+    if (DEBUG_MODE && servedLocally.size) {
+      console.log(
+        `Resolved ${servedLocally.size} of ${pkgList.length} crate(s) from the local Cargo registry.`,
+      );
+    }
+  }
+  const needsRegistry = pkgList.filter(
+    (p) =>
+      p?.name &&
+      p?.version &&
+      p.version !== "workspace" &&
+      !servedLocally.has(p),
+  );
   // Two URLs per crate — the crate document and its owners — fetched for the
   // whole list at once. The `workspace` guard below is mirrored here so the
   // batch does not request anything the loop would have skipped.
   const prefetched = await prefetchJson(
     prefetchEnabled()
-      ? pkgList
-          .filter((p) => p?.name && p?.version && p.version !== "workspace")
-          .flatMap((p) => [
-            { url: CRATES_URL + p.name },
-            { url: `${CRATES_URL + p.name}/owners` },
-          ])
+      ? needsRegistry.flatMap((p) => [
+          { url: CRATES_URL + p.name },
+          { url: `${CRATES_URL + p.name}/owners` },
+        ])
       : [],
   );
   for (const p of pkgList) {
@@ -1809,23 +1863,35 @@ export async function getCratesMetadata(pkgList) {
         cdepList.push(p);
         continue;
       }
+      if (servedLocally.has(p)) {
+        cdepList.push(p);
+        continue;
+      }
       if (DEBUG_MODE) {
         console.log(`Querying crates.io for ${p.name}@${p.version}`);
       }
       const crateUrl = CRATES_URL + p.name;
+      // A prefetch miss falls back to a direct request, which must queue
+      // behind the same crates.io gate the batch pool uses: the registry
+      // allows one request per second and does not care which code path in
+      // this process issued it.
       const res =
         prefetchedResponse(prefetched, crateUrl) ||
-        (await cdxgenAgent.get(crateUrl, {
-          responseType: "json",
-        }));
+        (await withHostRateLimit(crateUrl, () =>
+          cdxgenAgent.get(crateUrl, {
+            responseType: "json",
+          }),
+        ));
       let ownersRes;
       try {
         const ownersUrl = `${crateUrl}/owners`;
         ownersRes =
           prefetchedResponse(prefetched, ownersUrl) ||
-          (await cdxgenAgent.get(ownersUrl, {
-            responseType: "json",
-          }));
+          (await withHostRateLimit(ownersUrl, () =>
+            cdxgenAgent.get(ownersUrl, {
+              responseType: "json",
+            }),
+          ));
       } catch (_err) {
         ownersRes = undefined;
       }
@@ -1895,6 +1961,86 @@ export async function getCratesMetadata(pkgList) {
   return cdepList;
 }
 
+/**
+ * Apply the local Cargo registry's view of a crate to its component.
+ *
+ * Returns whether the component is complete enough to skip crates.io. The bar
+ * is the license: `getCratesMetadata` runs only under `FETCH_LICENSE`, so a
+ * crate whose license the local registry cannot name has not answered the
+ * question that was asked and still needs the network. That happens when the
+ * index knows a version but Cargo never extracted its sources — a crate the
+ * lockfile pins but the build never compiled.
+ *
+ * Publisher identity has no local equivalent, so a component served from here
+ * carries the publish time and yanked flag but none of the publisher-drift
+ * properties; `cdx:cargo:metadataSource` records which it was.
+ *
+ * @param {Object} p Package component to enrich in place.
+ * @returns {boolean} true when crates.io can be skipped for this crate.
+ */
+function applyCargoCacheMetadata(p) {
+  const local = readCargoCacheMetadata(p.name, p.version);
+  if (!local?.license) {
+    return false;
+  }
+  p.license = local.license;
+  if (local.description) {
+    p.description = local.description;
+  }
+  if (local.repository) {
+    p.repository = { url: local.repository };
+  }
+  if (local.homepage && local.homepage !== local.repository) {
+    p.homepage = { url: local.homepage };
+  }
+  if (!p._integrity && local.checksum) {
+    p._integrity = normalizeCargoIntegrity(local.checksum);
+  }
+  if (!p.properties) {
+    p.properties = [];
+  }
+  p.properties.push({
+    name: "cdx:cargo:metadataSource",
+    value: "local-registry",
+  });
+  if (local.rustVersion) {
+    p.properties.push({
+      name: "cdx:cargo:rust_version",
+      value: local.rustVersion,
+    });
+  }
+  if (local.latestVersion) {
+    p.properties.push({
+      name: "cdx:cargo:latest_version",
+      value: local.latestVersion,
+    });
+  }
+  if (typeof local.yanked === "boolean") {
+    p.properties.push({
+      name: "cdx:cargo:yanked",
+      value: String(local.yanked),
+    });
+  }
+  if (local.publishTime) {
+    p.properties.push({
+      name: "cdx:cargo:publishTime",
+      value: local.publishTime,
+    });
+  }
+  if (local.features && Object.keys(local.features).length) {
+    p.properties.push({
+      name: "cdx:cargo:features",
+      value: JSON.stringify(local.features),
+    });
+  }
+  // The download path is a documented, stable URL shape rather than something
+  // the API had to tell us, so it costs no request to state it.
+  p.distribution = {
+    url: `https://crates.io/api/v1/crates/${p.name}/${p.version}/download`,
+  };
+  return true;
+}
+
 /**
  * Method to retrieve metadata for dart packages by querying pub.dev
  *
@@ -2159,17 +2305,23 @@ export async function extractJarArchive(jarFile, tempDir, jarNSMapping = {}) {
           try {
             const sha = await checksumFile("sha1", jf);
             const searchurl = `https://central.sonatype.com/solrsearch/select?q=1:%22${sha}%22&rows=20&wt=json`;
-            const res = await cdxgenAgent.get(searchurl, {
-              responseType: "json",
-              timeout: {
-                lookup: 1000,
-                connect: 5000,
-                secureConnect: 5000,
-                socket: 1000,
-                send: 10000,
-                response: 1000,
-              },
-            });
+            // One Solr query per unidentified jar, against Central's origin
+            // rather than its CDN. A directory of jars issues these back to
+            // back, so they go through the same per-host gate the batch pool
+            // uses (500 ms, serial) instead of a second, ungoverned path.
+            const res = await withHostRateLimit(searchurl, () =>
+              cdxgenAgent.get(searchurl, {
+                responseType: "json",
+                timeout: {
+                  lookup: 1000,
+                  connect: 5000,
+                  secureConnect: 5000,
+                  socket: 1000,
+                  send: 10000,
+                  response: 1000,
+                },
+              }),
+            );
             const data = res?.body ? res.body["response"] : undefined;
             if (data && data["numFound"] === 1) {
               const jarInfo = data["docs"][0];
@@ -2471,6 +2623,12 @@ async function unresolvedJarComponents(unresolvedJars, jarFile, tempDir) {
   return components;
 }
 
+/**
+ * Property recording the NuGet release a versionless component's metadata was
+ * read from, so the description and licence in the output can be traced.
+ */
+const METADATA_VERSION_PROP = "cdx:nuget:metadata_version";
+
 async function getNugetUrl() {
   const req = "https://api.nuget.org/v3/index.json";
   const res = await cdxgenAgent.get(req, {
@@ -2546,60 +2704,46 @@ async function queryNuget(p, NUGET_URL) {
   if (!items?.[0]) {
     return [np, newBody, body];
   }
-  if (items[0] && !items[0].items) {
-    if (!p.version || p.version === "0.0.0" || p.version === "latest") {
-      const upper = items[items.length - 1].upper;
-      np.version = setLatestVersion(upper);
-    }
+  // A component that states no version at all is not the same as one carrying a
+  // `latest` or `0.0.0` placeholder. The placeholder asks to be replaced, so the
+  // registry's newest release becomes the component's version. A missing version
+  // means the manifest never said which release is in use - a `<Reference>` to an
+  // assembly is the usual source - and the newest release is then only a key for
+  // looking metadata up, never a version to write back. See #4359.
+  const versionStated = p.version && !["0.0.0", "latest"].includes(p.version);
+  let lookupVersion = p.version;
+  if (!versionStated) {
+    lookupVersion = setLatestVersion(items[items.length - 1].upper);
+    np.version = p.version ? lookupVersion : undefined;
+    np.metadataVersion = lookupVersion;
+  }
+  if (lookupVersion) {
     for (const item of items) {
-      if (np.version) {
-        const lower = compare(
-          coerce(item.lower, { loose: true }),
-          coerce(np.version, { loose: true }),
-        );
-        const upper = compare(
-          coerce(item.upper, { loose: true }),
-          coerce(np.version, { loose: true }),
-        );
-        if (lower !== 1 && upper !== -1) {
-          res = await cdxgenAgent.get(item["@id"], { responseType: "json" });
-          for (const i of res.body.items.reverse()) {
-            if (
-              i.catalogEntry &&
-              i.catalogEntry.version === coerceUp(np.version)
-            ) {
-              newBody.push(i);
-              return [np, newBody];
-            }
-          }
-        }
+      const lower = compare(
+        coerce(item.lower, { loose: true }),
+        coerce(lookupVersion, { loose: true }),
+      );
+      const upper = compare(
+        coerce(item.upper, { loose: true }),
+        coerce(lookupVersion, { loose: true }),
+      );
+      if (lower === 1 || upper === -1) {
+        continue;
       }
-    }
-  } else {
-    if (!p.version || p.version === "0.0.0" || p.version === "latest") {
-      const upper = items[items.length - 1].upper;
-      np.version = setLatestVersion(upper);
-    }
-    if (np.version) {
-      for (const item of items) {
-        const lower = compare(
-          coerce(item.lower, { loose: true }),
-          coerce(np.version, { loose: true }),
-        );
-        const upper = compare(
-          coerce(item.upper, { loose: true }),
-          coerce(np.version, { loose: true }),
-        );
-        if (lower !== 1 && upper !== -1) {
-          for (const i of item.items.reverse()) {
-            if (
-              i.catalogEntry &&
-              i.catalogEntry.version === coerceUp(np.version)
-            ) {
-              newBody.push(i);
-              return [np, newBody];
-            }
-          }
+      // The registration index either inlines the leaves of each page or points
+      // at a page that must be fetched.
+      let leaves = item.items;
+      if (!leaves) {
+        res = await cdxgenAgent.get(item["@id"], { responseType: "json" });
+        leaves = res.body.items;
+      }
+      for (const i of leaves.reverse()) {
+        if (
+          i.catalogEntry &&
+          i.catalogEntry.version === coerceUp(lookupVersion)
+        ) {
+          newBody.push(i);
+          return [np, newBody];
         }
       }
     }
@@ -2664,25 +2808,36 @@ export async function getNugetMetadata(pkgList, dependencies = undefined) {
           [np, newBody] = await queryNuget(p, NUGET_URL);
           if (p.version !== np.version) {
             const oldRef = p["bom-ref"];
-            p["bom-ref"] = decodeURIComponent(
-              build({
-                type: "nuget",
-                namespace: "" || null,
-                name: np.name,
-                version: np.version || null,
-              }),
-            );
-            depRepList[oldRef] = p["bom-ref"];
+            // The purl is rebuilt alongside the bom-ref. Leaving it behind let a
+            // component state one version in `version` and another - or none - in
+            // its purl, which is what a scanner range matches on. See #4359.
             p.version = np.version;
+            applyPurl(p, nugetPurl(np.name, np.version));
+            if (oldRef && p["bom-ref"] !== oldRef) {
+              depRepList[oldRef] = p["bom-ref"];
+            }
           }
           if (newBody && newBody.length > 0) {
             body = newBody[0];
           }
           if (body) {
             metadata_cache[cacheKey] = body;
-            // Set the latest version in case it is missing
-            if (!p.version && body.catalogEntry.version) {
-              p.version = body.catalogEntry.version;
+            // The component states no version, so the description and licence
+            // below were read from whichever release the lookup fell back to.
+            // That release is recorded rather than adopted as the version: a
+            // version cdxgen chose is not a version the project declared.
+            if (!p.version && np.metadataVersion) {
+              p.properties = p.properties || [];
+              if (
+                !p.properties.some(
+                  (prop) => prop.name === METADATA_VERSION_PROP,
+                )
+              ) {
+                p.properties.push({
+                  name: METADATA_VERSION_PROP,
+                  value: np.metadataVersion,
+                });
+              }
             }
             p.description = body.catalogEntry.description;
             if (body.catalogEntry.authors) {
@@ -2711,8 +2866,12 @@ export async function getNugetMetadata(pkgList, dependencies = undefined) {
             }
             if (body.catalogEntry.projectUrl) {
               p.repository = { url: body.catalogEntry.projectUrl };
+              // A versionless component gets the package's landing page. Pasting
+              // an undefined version into the path produced a 404 link.
               p.homepage = {
-                url: `https://www.nuget.org/packages/${p.name}/${p.version}/`,
+                url: p.version
+                  ? `https://www.nuget.org/packages/${p.name}/${p.version}/`
+                  : `https://www.nuget.org/packages/${p.name}/`,
               };
               if (
                 (!p.license || typeof p.license === "string") &&

package/lib/ecosystems/golem.js

@@ -1360,11 +1360,21 @@ function normalizedGolemDataFlowCallGraphMode(options = {}) {
   return callgraphMode;
 }
 
+function availableCpuCount() {
+  try {
+    return cpus()?.length || 1;
+  } catch {
+    // Deno denies the probe without --allow-sys=cpus; a worker-count hint
+    // must not abort the analysis, so fall back to a single worker.
+    return 1;
+  }
+}
+
 function appendGolemDataFlowArgs(args, dataFlowMode, options = {}) {
   if (dataFlowMode === "none") {
     return;
   }
-  const cpuCount = cpus()?.length || 1;
+  const cpuCount = availableCpuCount();
   const performanceWorkerCount = Math.max(1, Math.min(cpuCount, 4));
   const dataFlowCallgraph = normalizedGolemDataFlowCallGraphMode(options);
   const dataFlowPacks =