@cdxgen/cdxgen 13.2.0 → 13.3.0

package/lib/parsers/largeJson.js

@@ -0,0 +1,454 @@
+import { Buffer, constants } from "node:buffer";
+import { closeSync, openSync, readFileSync, readSync, statSync } from "node:fs";
+
+/**
+ * The largest file that `JSON.parse(readFileSync(file, "utf-8"))` can always
+ * decode. V8 caps a string at this many UTF-16 code units, and UTF-8 never
+ * decodes to more code units than it has bytes.
+ */
+export const MAX_JSON_TEXT_BYTES = constants.MAX_STRING_LENGTH;
+
+// A value at most this large is decoded with one JSON.parse. A larger object is
+// decoded member by member, and a larger array in runs of elements of about
+// this size, so no string ever holds more than one run.
+const BATCH_BYTES = 16 * 1024 * 1024;
+const CHUNK_BYTES = 8 * 1024 * 1024;
+
+const QUOTE = 0x22;
+const BACKSLASH = 0x5c;
+const COMMA = 0x2c;
+const COLON = 0x3a;
+const OPEN_BRACE = 0x7b;
+const CLOSE_BRACE = 0x7d;
+const OPEN_BRACKET = 0x5b;
+const CLOSE_BRACKET = 0x5d;
+
+const isWhitespace = (byte) =>
+  byte === 0x20 || byte === 0x0a || byte === 0x0d || byte === 0x09;
+
+// "-", a digit, or the first letter of true, false or null.
+const isScalarStart = (byte) =>
+  byte === 0x2d ||
+  (byte >= 0x30 && byte <= 0x39) ||
+  byte === 0x74 ||
+  byte === 0x66 ||
+  byte === 0x6e;
+
+// Assign like JSON.parse does: "__proto__" becomes an own property instead of
+// replacing the prototype.
+function defineMember(target, key, value) {
+  if (key === "__proto__") {
+    Object.defineProperty(target, key, {
+      configurable: true,
+      enumerable: true,
+      value,
+      writable: true,
+    });
+  } else {
+    target[key] = value;
+  }
+}
+
+function applySpec(value, spec) {
+  if (!spec || value === null || typeof value !== "object") {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    return spec.filter
+      ? value.filter((element) => spec.filter(element))
+      : value;
+  }
+  if (!spec.members) {
+    return value;
+  }
+  const picked = {};
+  for (const key of Object.keys(value)) {
+    if (Object.hasOwn(spec.members, key)) {
+      defineMember(picked, key, applySpec(value[key], spec.members[key]));
+    }
+  }
+  return picked;
+}
+
+/**
+ * Reads bytes [start, end) of an open file a chunk at a time, and can capture
+ * the bytes of a span that crosses chunk boundaries.
+ */
+class ByteCursor {
+  constructor(fd, start, end, chunkBytes) {
+    this.fd = fd;
+    this.end = end;
+    this.buf = Buffer.allocUnsafe(
+      Math.max(1, Math.min(chunkBytes, end - start)),
+    );
+    this.bufStart = start;
+    this.len = 0;
+    this.pos = 0;
+    this.captureFrom = -1;
+    this.parts = [];
+    this.partsBytes = 0;
+  }
+
+  get offset() {
+    return this.bufStart + this.pos;
+  }
+
+  fill() {
+    if (this.captureFrom >= 0) {
+      if (this.len > this.captureFrom) {
+        const part = Buffer.from(this.buf.subarray(this.captureFrom, this.len));
+        this.parts.push(part);
+        this.partsBytes += part.length;
+      }
+      this.captureFrom = 0;
+    }
+    this.bufStart += this.len;
+    const want = Math.min(this.buf.length, this.end - this.bufStart);
+    this.len =
+      want > 0 ? readSync(this.fd, this.buf, 0, want, this.bufStart) : 0;
+    this.pos = 0;
+    return this.len > 0;
+  }
+
+  peek() {
+    if (this.pos >= this.len && !this.fill()) {
+      return -1;
+    }
+    return this.buf[this.pos];
+  }
+
+  // The next byte that is not whitespace, left unconsumed; -1 at the end.
+  peekToken() {
+    for (;;) {
+      const byte = this.peek();
+      if (!isWhitespace(byte)) {
+        return byte;
+      }
+      this.pos++;
+    }
+  }
+
+  nextToken() {
+    const byte = this.peekToken();
+    if (byte >= 0) {
+      this.pos++;
+    }
+    return byte;
+  }
+
+  fail(message) {
+    throw new SyntaxError(`${message} at byte ${this.offset} of the JSON file`);
+  }
+
+  beginCapture() {
+    this.peek();
+    this.captureFrom = this.pos;
+    this.parts = [];
+    this.partsBytes = 0;
+  }
+
+  capturedBytes() {
+    return this.partsBytes + this.pos - this.captureFrom;
+  }
+
+  // The captured text up to `until`, an index into the current chunk.
+  endCapture(until = this.pos) {
+    const tail = this.buf.subarray(this.captureFrom, until);
+    const text = this.parts.length
+      ? Buffer.concat([...this.parts, tail]).toString("utf8")
+      : tail.toString("utf8");
+    this.captureFrom = -1;
+    this.parts = [];
+    this.partsBytes = 0;
+    return text;
+  }
+
+  // Move past one JSON value without decoding it. Brackets are balanced and
+  // strings are honoured, but the value is not otherwise validated; whatever
+  // decodes it later does that.
+  skipValue() {
+    const first = this.peekToken();
+    if (first === QUOTE) {
+      this.pos++;
+      this.skipStringBody();
+      return;
+    }
+    if (first === OPEN_BRACE || first === OPEN_BRACKET) {
+      this.skipContainer();
+      return;
+    }
+    if (!isScalarStart(first)) {
+      this.fail("Unexpected token");
+    }
+    // A number, true, false or null runs to the next delimiter.
+    for (;;) {
+      const byte = this.peek();
+      if (
+        byte < 0 ||
+        byte === COMMA ||
+        byte === CLOSE_BRACE ||
+        byte === CLOSE_BRACKET ||
+        isWhitespace(byte)
+      ) {
+        return;
+      }
+      this.pos++;
+    }
+  }
+
+  skipStringBody() {
+    let escaped = false;
+    for (;;) {
+      if (this.pos >= this.len && !this.fill()) {
+        this.fail("Unterminated string");
+      }
+      const { buf, len } = this;
+      for (let i = this.pos; i < len; i++) {
+        const byte = buf[i];
+        if (escaped) {
+          escaped = false;
+        } else if (byte === BACKSLASH) {
+          escaped = true;
+        } else if (byte === QUOTE) {
+          this.pos = i + 1;
+          return;
+        }
+      }
+      this.pos = len;
+    }
+  }
+
+  skipContainer() {
+    let depth = 0;
+    let inString = false;
+    let escaped = false;
+    for (;;) {
+      if (this.pos >= this.len && !this.fill()) {
+        this.fail("Unterminated container");
+      }
+      const { buf, len } = this;
+      for (let i = this.pos; i < len; i++) {
+        const byte = buf[i];
+        if (inString) {
+          if (escaped) {
+            escaped = false;
+          } else if (byte === BACKSLASH) {
+            escaped = true;
+          } else if (byte === QUOTE) {
+            inString = false;
+          }
+        } else if (byte === QUOTE) {
+          inString = true;
+        } else if (byte === OPEN_BRACE || byte === OPEN_BRACKET) {
+          depth++;
+        } else if (byte === CLOSE_BRACE || byte === CLOSE_BRACKET) {
+          depth--;
+          if (depth === 0) {
+            this.pos = i + 1;
+            return;
+          }
+        }
+      }
+      this.pos = len;
+    }
+  }
+}
+
+// The members of the object at the cursor, as { key, start, end } byte ranges
+// of their values, in file order. A repeated key keeps its first position and
+// its last range, which is what JSON.parse does.
+function scanObjectMembers(cursor) {
+  if (cursor.nextToken() !== OPEN_BRACE) {
+    cursor.fail("Expected an object");
+  }
+  const members = new Map();
+  if (cursor.peekToken() === CLOSE_BRACE) {
+    cursor.pos++;
+    return members;
+  }
+  for (;;) {
+    if (cursor.peekToken() !== QUOTE) {
+      cursor.fail("Expected a member name");
+    }
+    cursor.beginCapture();
+    cursor.skipValue();
+    const key = JSON.parse(cursor.endCapture());
+    if (cursor.nextToken() !== COLON) {
+      cursor.fail("Expected ':'");
+    }
+    cursor.peekToken();
+    const start = cursor.offset;
+    cursor.skipValue();
+    members.set(key, { start, end: cursor.offset });
+    const delimiter = cursor.nextToken();
+    if (delimiter === CLOSE_BRACE) {
+      return members;
+    }
+    if (delimiter !== COMMA) {
+      cursor.fail("Expected ',' or '}'");
+    }
+  }
+}
+
+// Decode the array at the cursor in runs of whole elements, keeping those that
+// `filter` accepts.
+function readArrayElements(cursor, filter, batchBytes) {
+  if (cursor.nextToken() !== OPEN_BRACKET) {
+    cursor.fail("Expected an array");
+  }
+  const elements = [];
+  if (cursor.peekToken() === CLOSE_BRACKET) {
+    cursor.pos++;
+    return elements;
+  }
+  const flush = () => {
+    // Everything captured but the delimiter just consumed.
+    for (const element of JSON.parse(
+      `[${cursor.endCapture(cursor.pos - 1)}]`,
+    )) {
+      if (!filter || filter(element)) {
+        elements.push(element);
+      }
+    }
+  };
+  cursor.beginCapture();
+  for (;;) {
+    cursor.skipValue();
+    const delimiter = cursor.nextToken();
+    if (delimiter === CLOSE_BRACKET) {
+      flush();
+      return elements;
+    }
+    if (delimiter !== COMMA) {
+      cursor.fail("Expected ',' or ']'");
+    }
+    if (cursor.capturedBytes() >= batchBytes) {
+      flush();
+      cursor.peekToken();
+      cursor.beginCapture();
+    }
+  }
+}
+
+function readBytes(fd, start, end) {
+  const bytes = Buffer.allocUnsafe(end - start);
+  let read = 0;
+  while (read < bytes.length) {
+    const count = readSync(fd, bytes, read, bytes.length - read, start + read);
+    if (!count) {
+      throw new SyntaxError("Unexpected end of the JSON file");
+    }
+    read += count;
+  }
+  return bytes.toString("utf8");
+}
+
+function decodeRange(fd, start, end, spec, options) {
+  if (end - start <= options.batchBytes) {
+    return applySpec(JSON.parse(readBytes(fd, start, end)), spec);
+  }
+  const cursor = new ByteCursor(fd, start, end, options.chunkBytes);
+  const first = cursor.peekToken();
+  let value;
+  if (first === OPEN_BRACE) {
+    value = {};
+    for (const [key, range] of scanObjectMembers(cursor)) {
+      if (spec?.members && !Object.hasOwn(spec.members, key)) {
+        continue;
+      }
+      defineMember(
+        value,
+        key,
+        decodeRange(fd, range.start, range.end, spec?.members?.[key], options),
+      );
+    }
+  } else if (first === OPEN_BRACKET) {
+    value = readArrayElements(cursor, spec?.filter, options.batchBytes);
+  } else {
+    // A string or number this large has no smaller parts; JSON.parse reports
+    // it the way it would for the whole file.
+    return JSON.parse(readBytes(fd, start, end));
+  }
+  if (cursor.peekToken() !== -1) {
+    cursor.fail("Unexpected data after the JSON value");
+  }
+  return value;
+}
+
+function withFile(filePath, callback) {
+  const fd = openSync(filePath, "r");
+  try {
+    return callback(fd);
+  } finally {
+    closeSync(fd);
+  }
+}
+
+function resolveOptions(options = {}) {
+  return {
+    batchBytes: options.batchBytes ?? BATCH_BYTES,
+    chunkBytes: options.chunkBytes ?? CHUNK_BYTES,
+  };
+}
+
+/**
+ * Locate the value of each member of the top-level object in a JSON file,
+ * without decoding any of them.
+ *
+ * @param {string} filePath JSON file whose top-level value is an object
+ * @param {Object} [options] Chunk size override, for tests
+ * @returns {Map<string, {start: number, end: number}>} Byte range of each member's value, in file order. A repeated key keeps its last value, as JSON.parse does.
+ * @throws {SyntaxError} When the top-level value is not a well-formed object
+ */
+export function indexJsonObject(filePath, options = {}) {
+  const { chunkBytes } = resolveOptions(options);
+  return withFile(filePath, (fd) => {
+    const cursor = new ByteCursor(fd, 0, statSync(filePath).size, chunkBytes);
+    const members = scanObjectMembers(cursor);
+    if (cursor.peekToken() !== -1) {
+      cursor.fail("Unexpected data after the JSON value");
+    }
+    return members;
+  });
+}
+
+/**
+ * Decode the JSON value stored in a byte range of a file, without ever holding
+ * more than a bounded run of it in one string.
+ *
+ * `spec` narrows what is kept. `{ members: { name: spec } }` keeps only the
+ * named members of an object, each narrowed by its own spec. `{ filter }`
+ * keeps the elements of an array that `filter(element)` accepts. Anything else
+ * keeps the whole value.
+ *
+ * @param {string} filePath JSON file
+ * @param {{start: number, end: number}} range Byte range of the value, such as one returned by indexJsonObject
+ * @param {Object} [spec] What to keep
+ * @param {Object} [options] Batch and chunk size overrides, for tests
+ * @returns {*} The decoded value
+ * @throws {SyntaxError} When the value is malformed
+ */
+export function readJsonRange(filePath, range, spec = undefined, options = {}) {
+  return withFile(filePath, (fd) =>
+    decodeRange(fd, range.start, range.end, spec, resolveOptions(options)),
+  );
+}
+
+/**
+ * Parse a JSON file of any size. A file that fits in one string is decoded
+ * with JSON.parse, and a larger one is streamed through readJsonRange, so the
+ * result is the same either way.
+ *
+ * @param {string} filePath JSON file
+ * @param {Object} [spec] What to keep, as for readJsonRange
+ * @param {Object} [options] Size overrides, for tests: maxTextBytes, batchBytes, chunkBytes
+ * @returns {*} The decoded value
+ * @throws {SyntaxError} When the file is malformed
+ */
+export function readJsonFile(filePath, spec = undefined, options = {}) {
+  const size = statSync(filePath).size;
+  if (size <= (options.maxTextBytes ?? MAX_JSON_TEXT_BYTES)) {
+    return applySpec(JSON.parse(readFileSync(filePath, "utf-8")), spec);
+  }
+  return readJsonRange(filePath, { start: 0, end: size }, spec, options);
+}

package/lib/parsers/npmrc.js

@@ -23,9 +23,24 @@ export const DEFAULT_NPMRC_BLOCKLIST = new Set([
   "rebuild-bundle",
 ]);
 
+/**
+ * Keys that collide with JavaScript object internals. A crafted .npmrc using
+ * them must never reach assignment on the result object: `result.__proto__`
+ * reads or replaces Object.prototype instead of creating an own property,
+ * which historically crashed the array branch with a TypeError.
+ *
+ * @type {Set<string>}
+ */
+const UNSAFE_NPMRC_KEYS = new Set(["__proto__", "constructor", "prototype"]);
+
 /**
  * Parse .npmrc content into a plain key-value object.
  *
+ * Keys that collide with JavaScript object internals are skipped: assigning
+ * them on a plain object either silently mutates Object.prototype semantics or,
+ * for the array syntax, crashed the parser with a TypeError because
+ * `result.__proto__` resolves to Object.prototype instead of an own property.
+ *
  * @param {string} content - Raw .npmrc file content
  * @returns {Object} Parsed key-value pairs
  */
@@ -46,6 +61,9 @@ export function parseNpmrc(content) {
     if (isArray) {
       key = key.slice(0, -2);
     }
+    if (UNSAFE_NPMRC_KEYS.has(key)) {
+      continue;
+    }
     if (
       (value.startsWith('"') && value.endsWith('"')) ||
       (value.startsWith("'") && value.endsWith("'"))
@@ -53,7 +71,7 @@ export function parseNpmrc(content) {
       value = value.slice(1, -1);
     }
     if (isArray) {
-      if (!result[key]) result[key] = [];
+      if (!Array.isArray(result[key])) result[key] = [];
       result[key].push(value);
     } else {
       result[key] = value;

package/lib/stages/postgen/postgen.js

@@ -22,6 +22,7 @@ import {
 } from "../../core/fs.js";
 import { thoughtLog } from "../../core/logger.js";
 import { dirNameStr } from "../../core/paths.js";
+import { sanitizeBomPropertyValue } from "../../core/propertySanitizer.js";
 import { CDXGEN_TOOL_GROUP, CDXGEN_TOOL_NAME } from "../../core/state.js";
 import {
   AI_INVENTORY_PROJECT_TYPES,
@@ -741,24 +742,18 @@ function applyReleaseNotes(bomJson, options, filePath) {
  */
 export function applyMetadata(bomJson, options) {
   if (!bomJson?.components) {
-    return bomJson;
+    return applyCustomMetadataProperties(bomJson, options);
   }
   const bomPkgTypes = new Set();
   const bomPkgNamespaces = new Set();
   const bomSrcFiles = new Set();
-  for (const comp of bomJson.components) {
-    if (comp.purl) {
-      try {
-        const purlObj = Purl.parse(comp.purl);
-        if (purlObj?.type) {
-          bomPkgTypes.add(purlObj.type);
-        }
-        if (purlObj?.namespace) {
-          bomPkgNamespaces.add(purlObj.namespace);
-        }
-      } catch (_e) {
-        // ignore
-      }
+  // Collect the manifests a component was identified from, rewriting any
+  // absolute path to a relative one as a side effect. Shared by the components
+  // and by metadata.component, which a collector such as the ruby one describes
+  // from the project's own gemspec (discussion 4410).
+  const collectSrcFiles = (comp) => {
+    if (!comp) {
+      return;
     }
     if (comp.properties) {
       for (const aprop of comp.properties) {
@@ -790,6 +785,10 @@ export function applyMetadata(bomJson, options) {
           Array.isArray(aidentityEvidence.methods)
         ) {
           for (const amethod of aidentityEvidence.methods) {
+            // `value` is optional: an attestation, for one, names no file.
+            if (typeof amethod?.value !== "string") {
+              continue;
+            }
             const rdir = relativeDir(amethod.value, options);
             if (
               comp.type !== "file" &&
@@ -806,7 +805,24 @@ export function applyMetadata(bomJson, options) {
         }
       }
     }
+  };
+  for (const comp of bomJson.components) {
+    if (comp.purl) {
+      try {
+        const purlObj = Purl.parse(comp.purl);
+        if (purlObj?.type) {
+          bomPkgTypes.add(purlObj.type);
+        }
+        if (purlObj?.namespace) {
+          bomPkgNamespaces.add(purlObj.namespace);
+        }
+      } catch (_e) {
+        // ignore
+      }
+    }
+    collectSrcFiles(comp);
   }
+  collectSrcFiles(bomJson.metadata?.component);
   if (!bomJson.metadata.properties) {
     bomJson.metadata.properties = [];
   }
@@ -862,6 +878,43 @@ export function applyMetadata(bomJson, options) {
       thoughtLog("BOM lacks package manifest details. Please help us improve!");
     }
   }
+  return applyCustomMetadataProperties(bomJson, options);
+}
+
+/**
+ * Add the metadata properties the user asked for on the command line, in the
+ * environment, or in a config file (discussion 4391), after whatever the scan
+ * itself discovered. They never replace a discovered property: the scan's own
+ * names, such as `cdx:bom:componentSrcFiles`, keep their values. A BOM without
+ * components still gets them.
+ *
+ * @param {Object} bomJson BOM JSON Object
+ * @param {Object} options CLI options, with `metadataProperties` as
+ *   `[{name, value}]`
+ *
+ * @returns {Object} BOM JSON Object
+ */
+function applyCustomMetadataProperties(bomJson, options) {
+  if (!bomJson?.metadata || !options?.metadataProperties?.length) {
+    return bomJson;
+  }
+  bomJson.metadata.properties = bomJson.metadata.properties || [];
+  for (const aprop of options.metadataProperties) {
+    if (!aprop?.name?.length || aprop.value === undefined) {
+      continue;
+    }
+    const value = sanitizeBomPropertyValue(aprop.name, `${aprop.value}`);
+    if (!value?.length) {
+      continue;
+    }
+    if (
+      !bomJson.metadata.properties.some(
+        (p) => p.name === aprop.name && p.value === value,
+      )
+    ) {
+      bomJson.metadata.properties.push({ name: aprop.name, value });
+    }
+  }
   return bomJson;
 }
 
@@ -1503,7 +1556,7 @@ export function annotate(bomJson, options) {
     for (const dep of bomJson.dependencies) {
       const newRef = `${bomLinkPrefix}${stripBomLink(bomJson.serialNumber, bomJson.version, dep.ref)}`;
       const newDependsOn = [];
-      for (const adon of dep.dependsOn) {
+      for (const adon of dep.dependsOn || []) {
         newDependsOn.push(
           `${bomLinkPrefix}${stripBomLink(bomJson.serialNumber, bomJson.version, adon)}`,
         );

package/lib/validator/bomValidator.js

@@ -15,7 +15,7 @@ import {
   toCycloneDxSpecVersionString,
 } from "../inventory/bomUtils.js";
 import { isPartialTree } from "../inventory/depsUtils.js";
-import { isValidPurl } from "../inventory/purl.js";
+import { isValidPurl, parsePurlLoose } from "../inventory/purl.js";
 
 const dirName = dirNameStr;
 // The v13 floor is spec 1.6. 1.4 and 1.5 were dropped here so that the JS and
@@ -790,17 +790,13 @@ export const validateRefs = (bomJson) => {
       if (!refMap[dep.ref]) {
         warningsList.push(`Invalid ref in dependencies ${dep.ref}`);
       }
-      let parentPurlType;
-      try {
-        const purlObj = Purl.parse(dep.ref);
-        parentPurlType = purlObj.type;
-      } catch (_e) {
-        // pass
-      }
+      // A ref is the decoded form of a purl, so a version with `+` does not
+      // parse strictly; the type is all that is needed here.
+      const parentPurlType = parsePurlLoose(dep.ref)?.type;
       if (
         parentComponentRef &&
         dep.ref === parentComponentRef &&
-        dep.dependsOn.length === 0 &&
+        !dep.dependsOn?.length &&
         bomJson.dependencies.length > 1
       ) {
         warningsList.push(
@@ -814,13 +810,7 @@ export const validateRefs = (bomJson) => {
               `Invalid ref in dependencies.dependsOn ${don}. Parent: ${dep.ref}`,
             );
           }
-          let childPurlType;
-          try {
-            const purlObj = Purl.parse(don);
-            childPurlType = purlObj.type;
-          } catch (_e) {
-            // pass
-          }
+          const childPurlType = parsePurlLoose(don)?.type;
           if (
             parentPurlType &&
             childPurlType &&