@cdxgen/cdxgen 13.2.0 → 13.3.0

package/lib/inventory/evidenceUtils.js

@@ -1,4 +1,3 @@
-import { readFileSync } from "node:fs";
 import { platform } from "node:os";
 import { extname } from "node:path";
 
@@ -6,7 +5,10 @@ import { build } from "@cdxgen/cdx-purl";
 
 import { safeExistsSync } from "../core/fs.js";
 import { thoughtLog } from "../core/logger.js";
-import { addDosaiReachabilityProperties } from "./dosai.js";
+import {
+  addDosaiReachabilityProperties,
+  readDosaiMethodsReport,
+} from "./dosai.js";
 import {
   addDosaiSetValue,
   buildDosaiPurlAliasMap,
@@ -275,10 +277,16 @@ export function addEvidenceForDotnet(pkgList, slicesFile) {
         });
     }
   }
-  let slicesData;
-  try {
-    slicesData = JSON.parse(readFileSync(slicesFile, "utf-8"));
-  } catch (_err) {
+  // Only MethodCalls into a package .dll produce evidence below, so a report
+  // too large to parse whole keeps just those.
+  const slicesData = readDosaiMethodsReport(slicesFile, {
+    keepMethodCall: (methodCall) =>
+      Boolean(
+        methodCall?.Module?.endsWith(".dll") &&
+          pkgFilePurlMap[methodCall.Module],
+      ),
+  });
+  if (!slicesData) {
     return pkgList;
   }
   if (slicesData && Object.keys(slicesData)) {

package/lib/inventory/fetchBatch.js

@@ -300,18 +300,126 @@ export function isSingleGateHost(host) {
   return isCratesHost(host) || host.toLowerCase() === "package.elm-lang.org";
 }
 
+/**
+ * Most URLs sent to one `cdxrs fetch` run. The envelope comes back as a single
+ * JSON string and V8 caps strings at ~512 MB, while a full npm packument is a
+ * few hundred KB on average and tens of MB for the largest packages. One run
+ * per workspace therefore outgrew the limit on large pnpm workspaces (issue
+ * 4393); runs of this size stay far below it for any realistic mix.
+ */
+const RUST_FETCH_CHUNK_SIZE = 250;
+
 /**
  * Run the batch through the `cdxrs fetch` subprocess and translate its
  * envelope into the shared result shape.
  *
+ * The URLs go out in chunks of {@link RUST_FETCH_CHUNK_SIZE}, one run at a time
+ * so each host still sees a single rate gate. A chunk whose envelope is still
+ * too large to deliver is split in half and retried, which is cheap because
+ * the first attempt filled the cdxrs disk cache. A chunk that fails for any
+ * other reason falls back to the JS pool on its own, so one bad run never
+ * costs the rest of the batch its Rust fetch.
+ *
  * @param {Array<{url: string, accept?: string, authRealm?: string}>} unique
  *   Already deduplicated by URL.
  * @param {Object} options Options with `timeoutMs`.
  * @returns {Promise<Map<string, BatchEntry>>}
  */
 async function rustBatchFetch(unique, options) {
+  const pending = [];
+  for (let i = 0; i < unique.length; i += RUST_FETCH_CHUNK_SIZE) {
+    pending.push(unique.slice(i, i + RUST_FETCH_CHUNK_SIZE));
+  }
+  const results = new Map();
+  const chunkStats = [];
+  while (pending.length) {
+    const chunk = pending.shift();
+    const outcome = await rustFetchChunk(chunk, options);
+    if (outcome.reason === "stdout-too-large" && chunk.length > 1) {
+      const middle = Math.ceil(chunk.length / 2);
+      pending.unshift(chunk.slice(0, middle), chunk.slice(middle));
+      continue;
+    }
+    if (!outcome.results) {
+      // Not "binary absent" (that never reaches here), so fall through to the
+      // JS pool rather than returning nothing: a transient cdxrs crash must
+      // not take this chunk's concurrency with it.
+      for (const [url, entry] of await jsBatchFetch(chunk)) {
+        results.set(url, entry);
+      }
+      chunkStats.push(_lastStats);
+      continue;
+    }
+    for (const [url, entry] of outcome.results) {
+      results.set(url, entry);
+    }
+    chunkStats.push(outcome.stats);
+  }
+  _lastStats = combineBatchStats(chunkStats);
+  if (DEBUG_MODE && _lastStats) {
+    const s = _lastStats;
+    console.log(
+      `cdxrs fetch: ${s.requests} url(s), ${s.unique} unique, ${s.ok} ok, ${s.failures} failed, ${s.cacheHits} cached, peak concurrency ${s.peakConcurrency}, ${s.elapsedMs} ms`,
+    );
+  }
+  return results;
+}
+
+/**
+ * Add up the stats of the runs that served one batch. Numeric fields sum
+ * because the runs are sequential, except peak concurrency, which is the
+ * widest run. Per-host entries under `hosts` are combined the same way.
+ *
+ * @param {Array<Object|null|undefined>} parts Stats from each run.
+ * @returns {Object|null} Combined stats, or null when no run reported any.
+ */
+function combineBatchStats(parts) {
+  const reported = parts.filter(Boolean);
+  if (!reported.length) {
+    return null;
+  }
+  if (reported.length === 1) {
+    return reported[0];
+  }
+  const combined = addStats({}, reported);
+  const hosts = {};
+  for (const part of reported) {
+    for (const [host, entry] of Object.entries(part.hosts || {})) {
+      hosts[host] = addStats(hosts[host] || {}, [entry]);
+    }
+  }
+  if (Object.keys(hosts).length) {
+    combined.hosts = hosts;
+  }
+  return combined;
+}
+
+function addStats(target, parts) {
+  for (const part of parts) {
+    for (const [key, value] of Object.entries(part)) {
+      if (typeof value !== "number") {
+        continue;
+      }
+      target[key] =
+        key === "peakConcurrency"
+          ? Math.max(target[key] || 0, value)
+          : (target[key] || 0) + value;
+    }
+  }
+  return target;
+}
+
+/**
+ * Run one chunk through `cdxrs fetch`.
+ *
+ * @param {Array<{url: string, accept?: string, authRealm?: string}>} chunk
+ * @param {Object} options Options with `timeoutMs`.
+ * @returns {Promise<{results?: Map<string, BatchEntry>, stats?: Object, reason?: string}>}
+ *   `results` when the run succeeded; otherwise the reason it did not.
+ */
+async function rustFetchChunk(chunk, options) {
   const payload = { requests: [] };
-  for (const request of unique) {
+  for (const request of chunk) {
     payload.requests.push({
       id: request.url,
       url: request.url,
@@ -328,20 +436,19 @@ async function rustBatchFetch(unique, options) {
   if (!ok) {
     if (DEBUG_MODE && reason !== "disabled" && reason !== "binary-not-found") {
       console.log(
-        `cdxrs fetch unavailable (${reason}); falling back to the JS batch pool.`,
+        reason === "stdout-too-large" && chunk.length > 1
+          ? `cdxrs fetch output for ${chunk.length} url(s) is too large; retrying in two halves.`
+          : `cdxrs fetch unavailable (${reason}); falling back to the JS batch pool.`,
       );
     }
-    // The Rust path failed in a way that is not "binary absent". Fall through
-    // to the JS pool rather than returning an empty map, so a transient cdxrs
-    // crash does not take the whole batch's concurrency with it.
-    return await jsBatchFetch(unique);
+    return { reason };
   }
 
   let envelope;
   try {
     envelope = JSON.parse(stdout);
   } catch (_err) {
-    return await jsBatchFetch(unique);
+    return { reason: "malformed-envelope" };
   }
   if (envelope?.schemaVersion !== BATCH_SCHEMA_VERSION) {
     // A binary from a different generation of this protocol is not something
@@ -351,10 +458,9 @@ async function rustBatchFetch(unique, options) {
         `cdxrs fetch envelope version ${envelope?.schemaVersion} != ${BATCH_SCHEMA_VERSION}; falling back to the JS batch pool.`,
       );
     }
-    return await jsBatchFetch(unique);
+    return { reason: "version-mismatch" };
   }
 
-  _lastStats = envelope.stats || null;
   const results = new Map();
   for (const result of envelope.results || []) {
     if (!result?.id) {
@@ -372,14 +478,7 @@ async function rustBatchFetch(unique, options) {
     const definite = status >= 400 && status < 500 && status !== 429;
     results.set(result.id, { ok: false, status, definite });
   }
-
-  if (DEBUG_MODE && envelope.stats) {
-    const s = envelope.stats;
-    console.log(
-      `cdxrs fetch: ${s.requests} url(s), ${s.unique} unique, ${s.ok} ok, ${s.failures} failed, ${s.cacheHits} cached, peak concurrency ${s.peakConcurrency}, ${s.elapsedMs} ms`,
-    );
-  }
-  return results;
+  return { results, stats: envelope.stats || null };
 }
 
 /**

package/lib/inventory/protobom.js

@@ -143,7 +143,14 @@ export const readBinary = (binFile, asJson, specVersion) => {
       ? decodeBomBinary(specVersion, binaryData, BINARY_READ_OPTIONS)
       : parseBomBinary(binaryData, BINARY_READ_OPTIONS);
   if (asJson) {
-    return encodeBomJson(bomObject);
+    const bomJson = encodeBomJson(bomObject);
+    // Protobuf omits empty repeated fields, so a round-trip drops empty
+    // dependsOn lists. Restore them so JSON and protobuf inputs yield
+    // identical documents.
+    for (const adep of bomJson?.dependencies || []) {
+      adep.dependsOn ??= [];
+    }
+    return bomJson;
   }
   return bomObject;
 };

package/lib/inventory/purl.js

@@ -53,15 +53,30 @@ export function purlTypeIsRegisteredString(purlString) {
 }
 
 /**
- * Encode a string for safe inclusion in a PackageURL, percent-encoding special characters
- * while preserving already-encoded `%40` sequences and keeping `:` and `/` unencoded.
+ * Encode a string for inclusion in a hand-written PackageURL string,
+ * percent-encoding every character the purl grammar does not allow literally,
+ * while preserving already-encoded `%40` sequences and keeping `:` and `/`
+ * unencoded.
+ *
+ * `encodeURIComponent` alone leaves `!`, `'`, `(`, `)` and `*` literal, and the
+ * purl grammar requires them escaped, so they are escaped here as well.
+ *
+ * Only for comparing against or assembling purl text. Never pass the result to
+ * `build()`, `new Purl()` or one of the helpers in this module: they escape
+ * their inputs themselves, and escaping twice turns `+` into `%252B`.
  *
  * @param {string} s String to encode
  * @returns {string} Encoded string suitable for use in a PackageURL component
  */
 export function encodeForPurl(s) {
   return s && !s.includes("%40")
-    ? encodeURIComponent(s).replace(/%3A/g, ":").replace(/%2F/g, "/")
+    ? encodeURIComponent(s)
+        .replace(
+          /[!'()*]/g,
+          (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`,
+        )
+        .replace(/%3A/g, ":")
+        .replace(/%2F/g, "/")
     : s;
 }
 
@@ -328,6 +343,140 @@ export function isValidPurl(candidate) {
   }
 }
 
+/**
+ * Split a purl into its components without requiring it to be well-formed.
+ *
+ * `Purl.parse` rejects a purl whose parts are not already escaped, which is the
+ * case this exists to handle: a `bom-ref` (cdxgen writes the decoded form of the
+ * purl there), a `go mod graph` token such as `github.com/Azure/go-autorest@v14.2.0+incompatible`,
+ * or a purl from a third-party document that left a `+` or a `://` unescaped.
+ * The separators (`:`, `/`, `@`, `?`, `#`, `&`, `=`) are structural and are read
+ * positionally; everything between them is decoded and handed back raw, ready
+ * for `build` to escape correctly.
+ *
+ * The version is taken after the last `@` of the path, so a scoped npm name
+ * (`@scope/name@1.0.0`) and a composer branch version (`dev-feature/x`) both
+ * split correctly.
+ *
+ * @param {string} purl Purl to split
+ * @returns {{type: string, namespace: (string|null), name: string, version: (string|null), qualifiers: (Object|null), subpath: (string|null)}|null} Parts, or null when the purl has no type and name
+ */
+export function splitPurlLoose(purl) {
+  if (typeof purl !== "string" || !purl.startsWith("pkg:")) {
+    return null;
+  }
+  const decode = (value) => {
+    try {
+      return decodeURIComponent(value);
+    } catch {
+      return value;
+    }
+  };
+  let rest = purl.slice("pkg:".length).replace(/^\/+/, "");
+  let subpath = null;
+  const hash = rest.indexOf("#");
+  if (hash !== -1) {
+    subpath = decode(rest.slice(hash + 1)) || null;
+    rest = rest.slice(0, hash);
+  }
+  let qualifiers = null;
+  const query = rest.indexOf("?");
+  if (query !== -1) {
+    qualifiers = {};
+    for (const pair of rest.slice(query + 1).split("&")) {
+      const equals = pair.indexOf("=");
+      if (equals <= 0) {
+        continue;
+      }
+      qualifiers[pair.slice(0, equals)] = decode(pair.slice(equals + 1));
+    }
+    if (!Object.keys(qualifiers).length) {
+      qualifiers = null;
+    }
+    rest = rest.slice(0, query);
+  }
+  const slash = rest.indexOf("/");
+  if (slash <= 0) {
+    return null;
+  }
+  const type = rest.slice(0, slash);
+  let path = rest.slice(slash + 1);
+  let version = null;
+  const at = path.lastIndexOf("@");
+  if (at > 0 && path[at - 1] !== "/") {
+    version = decode(path.slice(at + 1)) || null;
+    path = path.slice(0, at);
+  }
+  const segments = path.split("/").filter(Boolean).map(decode);
+  const name = segments.pop();
+  if (!name) {
+    return null;
+  }
+  return {
+    type,
+    namespace: segments.length ? segments.join("/") : null,
+    name,
+    version,
+    qualifiers,
+    subpath,
+  };
+}
+
+/**
+ * Parse a purl string that may carry unescaped reserved characters.
+ *
+ * A well-formed purl is parsed strictly, so its components come back exactly as
+ * cdx-purl reads them. Anything else is split structurally by
+ * {@link splitPurlLoose}. Use this where the input is known to be a decoded or
+ * hand-written purl (a `bom-ref`, a workspace reference, a package manager's
+ * `name@version` token), never to accept an invalid purl into a `purl` field:
+ * pass the parts to `build()` instead, which escapes them.
+ *
+ * @param {string} purlString Purl or decoded purl
+ * @returns {{type: string, namespace: (string|null), name: string, version: (string|null), qualifiers: (Object|null), subpath: (string|null)}|null} Raw components, or null when nothing purl-shaped remains
+ */
+export function parsePurlLoose(purlString) {
+  if (typeof purlString !== "string" || !purlString.startsWith("pkg:")) {
+    return null;
+  }
+  try {
+    const parsed = Purl.parse(purlString);
+    return {
+      type: parsed.type,
+      namespace: parsed.namespace || null,
+      name: parsed.name,
+      version: parsed.version || null,
+      qualifiers: parsed.qualifiers || null,
+      subpath: parsed.subpath || null,
+    };
+  } catch (err) {
+    if (!err?.code?.startsWith("E_")) {
+      throw err;
+    }
+  }
+  return splitPurlLoose(purlString);
+}
+
+/**
+ * Turn a decoded or hand-written purl into a canonical, correctly escaped one.
+ *
+ * `bom-ref` values are the decoded form of a purl
+ * (`pkg:npm/foo@1.0.0+build`), which cdx-purl rightly rejects as a purl. This
+ * recovers the purl the ref was made from by splitting it structurally and
+ * rebuilding it with `build()`, so every reserved character is escaped exactly
+ * once.
+ *
+ * @param {string} purlString Purl or decoded purl
+ * @returns {string|null} Canonical purl string, or `null` when no valid purl can be built
+ */
+export function canonicalPurlFromLooseString(purlString) {
+  const parts = parsePurlLoose(purlString);
+  if (!parts) {
+    return null;
+  }
+  return tryBuildPurl(parts);
+}
+
 /**
  * Parse a purl string, returning `null` instead of throwing when it is invalid.
  *

package/lib/inventory/sbomDocument.js

@@ -14,6 +14,7 @@
 import { build, Purl } from "@cdxgen/cdx-purl";
 
 import { isSpdxJsonLd } from "./bomUtils.js";
+import { splitPurlLoose } from "./purl.js";
 import { toCycloneDxLikeBom } from "./spdxUtils.js";
 
 // A document larger than this is not a plausible SBOM for a single
@@ -150,75 +151,6 @@ function encodeVersionBuildMetadata(purl) {
   return `${head.slice(0, at + 1)}${head.slice(at + 1).replaceAll("+", "%2B")}${tail}`;
 }
 
-/**
- * Split a purl into its components without requiring it to be well-formed.
- *
- * `Purl.parse` rejects a purl whose parts are not already escaped, which is the
- * case this exists to handle. The separators (`:`, `/`, `@`, `?`, `#`, `&`,
- * `=`) are structural and are read positionally; everything between them is
- * decoded and handed back raw, ready for `build` to escape correctly.
- *
- * @param {string} purl Purl to split
- * @returns {{type: string, namespace: (string|null), name: string, version: (string|null), qualifiers: (Object|null), subpath: (string|null)}|null} Parts, or null when the purl has no type and name
- */
-function splitPurl(purl) {
-  const decode = (value) => {
-    try {
-      return decodeURIComponent(value);
-    } catch {
-      return value;
-    }
-  };
-  let rest = purl.slice("pkg:".length).replace(/^\/+/, "");
-  let subpath = null;
-  const hash = rest.indexOf("#");
-  if (hash !== -1) {
-    subpath = decode(rest.slice(hash + 1)) || null;
-    rest = rest.slice(0, hash);
-  }
-  let qualifiers = null;
-  const query = rest.indexOf("?");
-  if (query !== -1) {
-    qualifiers = {};
-    for (const pair of rest.slice(query + 1).split("&")) {
-      const equals = pair.indexOf("=");
-      if (equals <= 0) {
-        continue;
-      }
-      qualifiers[pair.slice(0, equals)] = decode(pair.slice(equals + 1));
-    }
-    if (!Object.keys(qualifiers).length) {
-      qualifiers = null;
-    }
-    rest = rest.slice(0, query);
-  }
-  const slash = rest.indexOf("/");
-  if (slash <= 0) {
-    return null;
-  }
-  const type = rest.slice(0, slash);
-  let path = rest.slice(slash + 1);
-  let version = null;
-  const at = path.lastIndexOf("@");
-  if (at > 0) {
-    version = decode(path.slice(at + 1)) || null;
-    path = path.slice(0, at);
-  }
-  const segments = path.split("/").filter(Boolean).map(decode);
-  const name = segments.pop();
-  if (!name) {
-    return null;
-  }
-  return {
-    type,
-    namespace: segments.length ? segments.join("/") : null,
-    name,
-    version,
-    qualifiers,
-    subpath,
-  };
-}
-
 /**
  * Repair the purl of a component read from a third-party document.
  *
@@ -247,7 +179,7 @@ export function repairPurl(component) {
   } catch {
     // Fall through to the structural rebuild below.
   }
-  const parts = splitPurl(repaired);
+  const parts = splitPurlLoose(repaired);
   if (!parts) {
     return;
   }

package/lib/inventory/source.js

@@ -30,6 +30,7 @@ import {
   prefetchJson,
   withHostRateLimit,
 } from "./fetchBatch.js";
+import { canonicalPurlFromLooseString } from "./purl.js";
 
 /** Raised when a caller omits a registry helper that the purl type requires. */
 class MissingRegistryHelperError extends Error {}
@@ -954,6 +955,36 @@ function packageNameForLookup(purlObj) {
   return `${namespace}/${purlObj.name}`;
 }
 
+/**
+ * Read a purl supplied as a scan source.
+ *
+ * People type purls the way their package manager prints versions and URLs:
+ * `pkg:cargo/wasi@0.11.0+wasi-snapshot-preview1` or
+ * `pkg:npm/foo?vcs_url=git+https://...`. cdx-purl requires those `+` to be
+ * escaped, so a purl that does not parse strictly is split on its separators
+ * and rebuilt with every component escaped once. The rebuilt purl is then
+ * parsed strictly, so validation always inspects cdx-purl's reading of it.
+ *
+ * @param {string} purlString Package URL as supplied
+ * @returns {Purl|undefined} Parsed purl, or undefined when none can be recovered
+ */
+function parsePurlSourceInput(purlString) {
+  try {
+    return Purl.parse(purlString);
+  } catch {
+    // Fall through to the structural rebuild.
+  }
+  const canonical = canonicalPurlFromLooseString(purlString);
+  if (!canonical) {
+    return undefined;
+  }
+  try {
+    return Purl.parse(canonical);
+  } catch {
+    return undefined;
+  }
+}
+
 /**
  * Validate package URL source input and return an error object when invalid.
  *
@@ -964,10 +995,8 @@ export function validatePurlSource(purlString) {
   if (!maybePurlSource(purlString)) {
     return null;
   }
-  let purlObj;
-  try {
-    purlObj = Purl.parse(purlString);
-  } catch (_err) {
+  const purlObj = parsePurlSourceInput(purlString);
+  if (!purlObj) {
     return {
       status: 400,
       error: "Invalid purl source",
@@ -1209,24 +1238,13 @@ function parsePurlForSourceResolution(purlString) {
   if (!maybePurlSource(purlString) || validatePurlSource(purlString)) {
     return undefined;
   }
-  let purlObj;
-  try {
-    // Some purls in the wild use unencoded + in qualifier values (e.g.
-    // git+https:// in vcs_url). cdx-purl requires percent-encoding; normalize
-    // the qualifier section before parsing so real-world purls resolve.
-    const qIdx = purlString.indexOf("?");
-    const safePurlString =
-      qIdx >= 0
-        ? purlString.slice(0, qIdx + 1) +
-          purlString.slice(qIdx + 1).replaceAll("+", "%2B")
-        : purlString;
-    purlObj = Purl.parse(safePurlString);
-  } catch (err) {
+  // Purls in the wild leave `+` unescaped in versions and in qualifier values
+  // such as `vcs_url=git+https://`; the shared reader rebuilds them.
+  const purlObj = parsePurlSourceInput(purlString);
+  if (!purlObj) {
     if (DEBUG_MODE) {
-      const errorMessage = err?.message || String(err);
       thoughtLog("Unable to resolve repository URL for purl.", {
         purlString,
-        errorMessage,
       });
     }
     return undefined;