@cdxgen/cdxgen 13.1.0 → 13.2.0

package/lib/parsers/onnx.js

@@ -0,0 +1,294 @@
+import { closeSync, openSync, readSync, statSync } from "node:fs";
+
+// An ONNX model is a protobuf-encoded ModelProto. Only the header fields are
+// read here; the graph payload is skipped by offset, so inspection stays
+// cheap for multi-gigabyte files and never loads tensors.
+//
+// ModelProto fields (https://onnx.ai/onnx/repo-docs/Model.html):
+//   1 ir_version (varint), 2 producer_name (string),
+//   3 producer_version (string), 4 domain (string),
+//   5 model_version (varint), 6 doc_string (string),
+//   7 graph (message, skipped), 8 opset_import (repeated message).
+// Each opset_import entry is an OperatorSetIdProto:
+//   1 domain (string), 2 version (varint).
+
+/** Fields are walked from the front; a hostile file cannot force a scan
+ * deeper than this many top-level entries. */
+const MAX_TOP_LEVEL_FIELDS = 128;
+/** Largest single field payload that will actually be read into memory. */
+const MAX_FIELD_BYTES = 4096;
+
+/**
+ * Read the identifying header fields of an ONNX model file.
+ *
+ * Only `ir_version`, `producer_name`, `producer_version`, `model_version`,
+ * and the `opset_import` entries are extracted. The graph and every other
+ * payload are skipped by offset without being read, so no tensor data is
+ * ever loaded and no model code is executed.
+ *
+ * @param {string} filePath Path to the `.onnx` file
+ * @returns {{
+ *   irVersion: number,
+ *   producerName: string,
+ *   producerVersion: string,
+ *   modelVersion: number,
+ *   opsets: Array<{domain: string, version: number}>,
+ * } | undefined} Header summary, or undefined when it cannot be read
+ */
+export function readOnnxHeader(filePath) {
+  let fd;
+  try {
+    const stats = statSync(filePath);
+    if (!stats.isFile() || stats.size < 2) {
+      return undefined;
+    }
+    fd = openSync(filePath, "r");
+    // The whole file is not buffered; fields are fetched by offset. A small
+    // sliding buffer is re-used for varints and short strings.
+    const reader = new OffsetReader(fd, stats.size);
+    const result = {
+      irVersion: 0,
+      producerName: "",
+      producerVersion: "",
+      modelVersion: 0,
+      opsets: [],
+    };
+    let fieldsRead = 0;
+    for (let i = 0; i < MAX_TOP_LEVEL_FIELDS; i += 1) {
+      const tag = reader.readVarint();
+      if (tag === undefined) {
+        break;
+      }
+      const fieldNumber = Number(tag >> 3n);
+      const wireType = Number(tag & 7n);
+      if (fieldNumber === 0) {
+        break;
+      }
+      if (wireType === 0) {
+        const value = reader.readVarint();
+        if (value === undefined) {
+          break;
+        }
+        fieldsRead += 1;
+        if (fieldNumber === 1) {
+          result.irVersion = Number(value);
+        } else if (fieldNumber === 5) {
+          result.modelVersion = Number(value);
+        }
+        continue;
+      }
+      if (wireType !== 2) {
+        // Group and fixed-width wire types do not appear in ModelProto at the
+        // top level; stop rather than guess their length.
+        break;
+      }
+      const length = reader.readVarint();
+      if (length === undefined) {
+        break;
+      }
+      const byteLength = Number(length);
+      if (!Number.isSafeInteger(byteLength) || byteLength < 0) {
+        break;
+      }
+      if (fieldNumber === 8 && byteLength <= MAX_FIELD_BYTES) {
+        const payload = reader.readBytes(byteLength);
+        if (!payload) {
+          break;
+        }
+        const opset = parseOperatorSetEntry(payload);
+        if (opset) {
+          result.opsets.push(opset);
+        }
+        fieldsRead += 1;
+        continue;
+      }
+      if (
+        (fieldNumber === 2 || fieldNumber === 3) &&
+        byteLength <= MAX_FIELD_BYTES
+      ) {
+        const payload = reader.readBytes(byteLength);
+        if (!payload) {
+          break;
+        }
+        if (fieldNumber === 2) {
+          result.producerName = payload.toString("utf-8");
+        } else {
+          result.producerVersion = payload.toString("utf-8");
+        }
+        fieldsRead += 1;
+        continue;
+      }
+      // Skip anything else (notably the graph) by offset.
+      if (!reader.skip(byteLength)) {
+        break;
+      }
+      fieldsRead += 1;
+    }
+    // A real ModelProto always carries at least an ir_version; zero consumed
+    // fields means the file is not an ONNX model at all.
+    if (fieldsRead === 0) {
+      return undefined;
+    }
+    return result;
+  } catch {
+    return undefined;
+  } finally {
+    if (fd !== undefined) {
+      closeSync(fd);
+    }
+  }
+}
+
+/**
+ * Reader that fetches bytes from a file descriptor by absolute offset.
+ *
+ * @param {number} fd Open file descriptor
+ * @param {number} fileSize Total file size
+ */
+class OffsetReader {
+  constructor(fd, fileSize) {
+    this.fd = fd;
+    this.fileSize = fileSize;
+    this.offset = 0;
+    this.buffer = Buffer.alloc(16);
+  }
+
+  /**
+   * Read a protobuf varint at the cursor.
+   *
+   * @returns {bigint|undefined} Value, or undefined at end of input
+   */
+  readVarint() {
+    let result = 0n;
+    let shift = 0n;
+    for (let i = 0; i < 10; i += 1) {
+      if (!this.fillBuffer(1)) {
+        return undefined;
+      }
+      const byte = this.buffer[0];
+      this.offset += 1;
+      result |= BigInt(byte & 0x7f) << shift;
+      if ((byte & 0x80) === 0) {
+        return result;
+      }
+      shift += 7n;
+    }
+    return undefined;
+  }
+
+  /**
+   * Read a length-delimited payload at the cursor.
+   *
+   * @param {number} length Payload length
+   * @returns {Buffer|undefined} Payload, or undefined when unavailable
+   */
+  readBytes(length) {
+    const payload = Buffer.alloc(length);
+    const bytesRead = readSync(this.fd, payload, 0, length, this.offset);
+    if (bytesRead !== length) {
+      return undefined;
+    }
+    this.offset += length;
+    return payload;
+  }
+
+  /**
+   * Advance the cursor without reading the payload.
+   *
+   * @param {number} length Bytes to skip
+   * @returns {boolean} true when the skip stayed within the file
+   */
+  skip(length) {
+    const next = this.offset + length;
+    if (next > this.fileSize) {
+      return false;
+    }
+    this.offset = next;
+    return true;
+  }
+
+  /**
+   * Ensure up to `length` bytes are available in the scratch buffer.
+   *
+   * @param {number} length Bytes needed
+   * @returns {boolean} true when the read succeeded
+   */
+  fillBuffer(length) {
+    if (this.offset + length > this.fileSize) {
+      return false;
+    }
+    const bytesRead = readSync(this.fd, this.buffer, 0, length, this.offset);
+    return bytesRead === length;
+  }
+}
+
+/**
+ * Parse one OperatorSetIdProto payload.
+ *
+ * @param {Buffer} payload Field payload
+ * @returns {{domain: string, version: number}|undefined} Parsed entry
+ */
+function parseOperatorSetEntry(payload) {
+  let pos = 0;
+  let domain = "";
+  let version = 0;
+  while (pos < payload.length) {
+    const tag = readVarintFrom(payload, pos);
+    if (!tag) {
+      break;
+    }
+    pos = tag.next;
+    const fieldNumber = Number(tag.value >> 3n);
+    const wireType = Number(tag.value & 7n);
+    if (wireType === 0) {
+      const value = readVarintFrom(payload, pos);
+      if (!value) {
+        break;
+      }
+      pos = value.next;
+      if (fieldNumber === 2) {
+        version = Number(value.value);
+      }
+    } else if (wireType === 2) {
+      const length = readVarintFrom(payload, pos);
+      if (!length) {
+        break;
+      }
+      pos = length.next;
+      const byteLength = Number(length.value);
+      if (pos + byteLength > payload.length) {
+        break;
+      }
+      if (fieldNumber === 1) {
+        domain = payload.toString("utf-8", pos, pos + byteLength);
+      }
+      pos += byteLength;
+    } else {
+      break;
+    }
+  }
+  return { domain, version };
+}
+
+/**
+ * Read a varint from a buffer at a position.
+ *
+ * @param {Buffer} buffer Source buffer
+ * @param {number} pos Start position
+ * @returns {{value: bigint, next: number}|undefined} Result, or undefined
+ */
+function readVarintFrom(buffer, pos) {
+  let value = 0n;
+  let shift = 0n;
+  let cursor = pos;
+  for (let i = 0; i < 10 && cursor < buffer.length; i += 1) {
+    const byte = buffer[cursor];
+    cursor += 1;
+    value |= BigInt(byte & 0x7f) << shift;
+    if ((byte & 0x80) === 0) {
+      return { value, next: cursor };
+    }
+    shift += 7n;
+  }
+  return undefined;
+}

package/lib/parsers/safetensors.js

@@ -0,0 +1,114 @@
+import { closeSync, openSync, readSync, statSync } from "node:fs";
+
+// A safetensors file begins with an 8-byte little-endian unsigned integer
+// giving the byte length of a JSON header, followed by that JSON header, and
+// then the raw tensor buffers. Only the header is parsed here: tensor data is
+// never read, so inspection stays cheap and side-effect free.
+const HEADER_LENGTH_BYTES = 8;
+// A header longer than this almost certainly indicates a non-safetensors file
+// or a corrupt/hostile input; refuse to allocate for it.
+const MAX_HEADER_BYTES = 100 * 1024 * 1024;
+
+/**
+ * Read and parse the JSON header of a safetensors model file.
+ *
+ * Only the leading header is read; tensor payloads are ignored. The returned
+ * object separates the optional `__metadata__` map from the per-tensor
+ * descriptors.
+ *
+ * @param {string} filePath Path to a `.safetensors` file
+ * @returns {{
+ *   metadata: Object,
+ *   tensorCount: number,
+ *   dtypes: string[],
+ *   totalParameters: number,
+ * } | undefined} Parsed header summary, or undefined when it cannot be read
+ */
+export function readSafetensorsHeader(filePath) {
+  let fd;
+  try {
+    const stats = statSync(filePath);
+    if (stats.size < HEADER_LENGTH_BYTES) {
+      return undefined;
+    }
+    fd = openSync(filePath, "r");
+    const lengthBuffer = Buffer.alloc(HEADER_LENGTH_BYTES);
+    readSync(fd, lengthBuffer, 0, HEADER_LENGTH_BYTES, 0);
+    const headerLength = Number(lengthBuffer.readBigUInt64LE(0));
+    if (
+      !Number.isSafeInteger(headerLength) ||
+      headerLength <= 0 ||
+      headerLength > MAX_HEADER_BYTES ||
+      headerLength + HEADER_LENGTH_BYTES > stats.size
+    ) {
+      return undefined;
+    }
+    const headerBuffer = Buffer.alloc(headerLength);
+    readSync(fd, headerBuffer, 0, headerLength, HEADER_LENGTH_BYTES);
+    const header = JSON.parse(headerBuffer.toString("utf-8"));
+    if (!header || typeof header !== "object") {
+      return undefined;
+    }
+    return summarizeHeader(header);
+  } catch {
+    return undefined;
+  } finally {
+    if (fd !== undefined) {
+      try {
+        closeSync(fd);
+      } catch {
+        // ignore close failures
+      }
+    }
+  }
+}
+
+/**
+ * Summarize a parsed safetensors header into metadata and tensor statistics.
+ *
+ * @param {Object} header Parsed safetensors JSON header
+ * @returns {{
+ *   metadata: Object,
+ *   tensorCount: number,
+ *   dtypes: string[],
+ *   totalParameters: number,
+ * }} Header summary
+ */
+function summarizeHeader(header) {
+  const metadata =
+    header.__metadata__ && typeof header.__metadata__ === "object"
+      ? header.__metadata__
+      : {};
+  const dtypes = new Set();
+  let tensorCount = 0;
+  let totalParameters = 0;
+  for (const [key, descriptor] of Object.entries(header)) {
+    if (
+      key === "__metadata__" ||
+      !descriptor ||
+      typeof descriptor !== "object"
+    ) {
+      continue;
+    }
+    tensorCount++;
+    if (typeof descriptor.dtype === "string") {
+      dtypes.add(descriptor.dtype);
+    }
+    if (Array.isArray(descriptor.shape) && descriptor.shape.length) {
+      const elements = descriptor.shape.reduce(
+        (product, dimension) =>
+          Number.isFinite(dimension) ? product * dimension : product,
+        1,
+      );
+      if (Number.isFinite(elements)) {
+        totalParameters += elements;
+      }
+    }
+  }
+  return {
+    metadata,
+    tensorCount,
+    dtypes: Array.from(dtypes).sort(),
+    totalParameters,
+  };
+}

package/lib/stages/postgen/introspection/markers.js

@@ -28,8 +28,9 @@ const MAX_SUBDIRS_SCANNED = 100;
  * Marker table: ecosystem → manifest and lockfile names that indicate the
  * ecosystem is part of the project. Entries are either exact file names or
  * `{ suffix }` matchers for extension-driven ecosystems. Ecosystems cdxgen
- * cannot parse (elm, crystal, nim, perl, r) are included so the reflection
- * can report them as coverage gaps instead of silently ignoring them.
+ * cannot parse (currently only dub, the D package manager) are included so
+ * the reflection can report them as coverage gaps instead of silently
+ * ignoring them.
  *
  * @type {Readonly<Record<string, {names: string[], suffixes: string[]}>>}
  */
@@ -103,18 +104,40 @@ export const ECOSYSTEM_MARKERS = Object.freeze({
   swift: { names: ["Package.swift", "Package.resolved"], suffixes: [] },
   elixir: { names: ["mix.exs", "mix.lock"], suffixes: [] },
   c: {
-    names: ["CMakeLists.txt", "conanfile.txt", "conanfile.py"],
+    names: [
+      "CMakeLists.txt",
+      "conanfile.txt",
+      "conanfile.py",
+      "xmake-requires.lock",
+    ],
     suffixes: [],
   },
-  haskell: { names: ["cabal.project", "stack.yaml"], suffixes: [".cabal"] },
-  elm: { names: ["elm.json"], suffixes: [] },
+  haskell: {
+    names: ["cabal.project", "stack.yaml", "stack.yaml.lock", "stack.lock"],
+    suffixes: [".cabal"],
+  },
+  elm: { names: ["elm.json", "elm-package.json"], suffixes: [] },
+  dub: { names: ["dub.json", "dub.sdl"], suffixes: [] },
   crystal: { names: ["shard.yml", "shard.lock"], suffixes: [] },
   nim: { names: ["nim.cfg", "config.nims"], suffixes: [".nimble"] },
   perl: {
-    names: ["cpanfile", "META.json", "META.yml", "Makefile.PL", "Build.PL"],
+    names: [
+      "cpanfile",
+      "cpanfile.snapshot",
+      "META.json",
+      "META.yml",
+      "Makefile.PL",
+      "Build.PL",
+    ],
     suffixes: [],
   },
   r: { names: ["DESCRIPTION", "renv.lock"], suffixes: [] },
+  julia: { names: ["Project.toml", "Manifest.toml"], suffixes: [] },
+  erlang: { names: ["rebar.config", "rebar.lock"], suffixes: [] },
+  ocaml: { names: ["dune-project"], suffixes: [".opam"] },
+  lua: { names: ["luarocks.lock"], suffixes: [".rockspec"] },
+  spack: { names: ["spack.yaml", "spack.lock"], suffixes: [] },
+  terraform: { names: [".terraform.lock.hcl"], suffixes: [] },
 });
 
 /** Marker file name → ecosystems claiming it, and marker suffix → claiming

package/lib/stages/postgen/postgen.js

@@ -565,11 +565,21 @@ export async function postProcess(bomNSData, options, filePath) {
   bomNSData.bomJson = applyMetadata(bomNSData.bomJson, options);
   bomNSData.bomJson = applyLicenseEnhancement(bomNSData.bomJson, options);
   bomNSData.bomJson = applyContainerInventoryMetadata(bomNSData.bomJson);
+  const buildOnlyComponents = extractBuildOnlyComponents(
+    bomNSData.bomJson,
+    options,
+  );
   bomNSData.bomJson = applyFormulation(
     bomNSData.bomJson,
     options,
     filePath,
-    bomNSData.formulationList,
+    buildOnlyComponents.length
+      ? [...(bomNSData.formulationList || []), ...buildOnlyComponents]
+      : bomNSData.formulationList,
+  );
+  bomNSData.bomJson = applyBuildOnlyFormulation(
+    bomNSData.bomJson,
+    buildOnlyComponents,
   );
   bomNSData.bomJson = applyReleaseNotes(bomNSData.bomJson, options, filePath);
   bomNSData.bomJson = applyComponentTypeFilter(bomNSData.bomJson, options);
@@ -964,6 +974,154 @@ function getIdentityTechniques(comp) {
   return techniques;
 }
 
+/**
+ * Ensure the components moved out of the assembly have landed in the
+ * formulation section.
+ *
+ * With `--include-formulation` they were handed to the full formulation pass.
+ * Without it there is no formulation section, so a minimal one is created for
+ * them alone: moving a component out of `components[]` and then dropping it
+ * would lose it from the document.
+ *
+ * @param {Object} bomJson BOM JSON object
+ * @param {Object[]} buildOnlyComponents Components removed from `components[]`
+ *
+ * @returns {Object} The same `bomJson`
+ */
+function applyBuildOnlyFormulation(bomJson, buildOnlyComponents) {
+  if (!buildOnlyComponents?.length || !bomJson) {
+    return bomJson;
+  }
+  const alreadyPlaced = (bomJson.formulation || []).some((formula) =>
+    formula?.components?.some((comp) =>
+      buildOnlyComponents.some((moved) => moved["bom-ref"] === comp["bom-ref"]),
+    ),
+  );
+  if (alreadyPlaced) {
+    return bomJson;
+  }
+  if (!bomJson.formulation) {
+    bomJson.formulation = [];
+  }
+  bomJson.formulation.push({
+    "bom-ref": `${bomJson.metadata?.component?.["bom-ref"] || "cdxgen"}#build-dependencies`,
+    components: buildOnlyComponents,
+  });
+  return bomJson;
+}
+
+/**
+ * Report whether a component exists only to produce the build, rather than
+ * being part of the delivered assembly. A cargo build dependency and a
+ * proc-macro crate are both marked with `cdx:cargo:hostOnly`.
+ *
+ * @param {Object} comp Component
+ * @returns {boolean} True for a build-time-only component
+ */
+function isBuildOnlyComponent(comp) {
+  return !!comp?.properties?.some(
+    (property) =>
+      property.name === "cdx:cargo:hostOnly" && property.value === "true",
+  );
+}
+
+/**
+ * Move the build-time-only components out of `components[]`, which describes
+ * the delivered assembly, and hand them to the formulation section, which is
+ * where CycloneDX describes how the assembly was produced.
+ *
+ * Their bom-refs stay valid because the formulation components live in the same
+ * document, so the dependency graph is left intact. `formulation` exists from
+ * CycloneDX 1.5, so an older document keeps them in `components[]` with their
+ * `cdx:cargo:hostOnly` marker rather than losing them.
+ *
+ * @param {Object} bomJson BOM JSON object
+ * @param {Object} options CLI options
+ *
+ * @returns {Object[]} The components removed from `components[]`
+ */
+export function extractBuildOnlyComponents(bomJson, options) {
+  if (options?.specVersion < 1.5 || !bomJson?.components?.length) {
+    return [];
+  }
+  const buildOnlyComponents = [];
+  const retainedComponents = [];
+  for (const comp of bomJson.components) {
+    if (isBuildOnlyComponent(comp)) {
+      buildOnlyComponents.push(comp);
+    } else {
+      retainedComponents.push(comp);
+    }
+  }
+  if (buildOnlyComponents.length) {
+    bomJson.components = retainedComponents;
+  }
+  return buildOnlyComponents;
+}
+
+/**
+ * Report whether a component carries evidence that it is actually used: an
+ * occurrence in the analyzed sources, or a frame in a callstack.
+ *
+ * @param {Object} comp Component
+ * @returns {boolean} True when usage evidence is attached
+ */
+function hasUsageEvidence(comp) {
+  if (comp?.evidence?.occurrences?.length) {
+    return true;
+  }
+  return !!comp?.evidence?.callstack?.frames?.length;
+}
+
+/**
+ * Re-apply the `--required-only` filter once the evidence stage has attached
+ * occurrence and callstack data.
+ *
+ * A manifest can only say that a dependency is optional; the evidence says
+ * whether it is reached. A component the analyzers observed in the sources is
+ * promoted to `required` so it survives the filter, while an optional component
+ * with no observed usage is dropped as before. A component scoped `excluded` -
+ * a dev or test dependency - keeps that scope: an occurrence inside the test
+ * sources is not evidence that it ships.
+ *
+ * @param {Object} bomJson BOM JSON object carrying evidence
+ * @param {Object} options CLI options
+ *
+ * @returns {Object} Filtered BOM JSON
+ */
+export function applyEvidenceBasedFilter(bomJson, options) {
+  if (!options?.requiredOnly || !bomJson?.components?.length) {
+    return bomJson;
+  }
+  if (!(options.evidence || options.includeCrypto)) {
+    return bomJson;
+  }
+  for (const comp of bomJson.components) {
+    if (!hasUsageEvidence(comp)) {
+      continue;
+    }
+    if (!comp.properties) {
+      comp.properties = [];
+    }
+    const observed = comp.evidence?.occurrences?.length
+      ? "occurrence"
+      : "callstack";
+    if (
+      !comp.properties.some(
+        (property) => property.name === "cdx:evidence:usage",
+      )
+    ) {
+      comp.properties.push({ name: "cdx:evidence:usage", value: observed });
+    }
+    if (comp.scope === "optional") {
+      comp.scope = "required";
+    }
+  }
+  // `evidenceFilterStage` tells filterBom that the evidence is in hand, so the
+  // scope filter it deferred earlier must run now.
+  return filterBom(bomJson, { ...options, evidenceFilterStage: true });
+}
+
 /**
  * Filter BOM based on options
  *
@@ -983,6 +1141,15 @@ export function filterBom(bomJson, options) {
   const allowedComponentTypes = new Set(
     normalizeCycloneDxComponentTypeFilter(options?.componentType),
   );
+  // Evidence generation runs after this stage, so dropping the optional
+  // components here would deny the analyzers the chance to prove that one is
+  // used. The scope filter is re-applied by applyEvidenceBasedFilter once the
+  // occurrence and callstack evidence is attached.
+  const deferRequiredOnly = !!(
+    options.requiredOnly &&
+    !options.evidenceFilterStage &&
+    (options.evidence || options.includeCrypto)
+  );
   for (const comp of bomJson.components) {
     if (shouldExcludeInventoryType(comp, options)) {
       filtered = true;
@@ -1025,6 +1192,7 @@ export function filterBom(bomJson, options) {
     }
     if (
       options.requiredOnly &&
+      !deferRequiredOnly &&
       comp.scope &&
       ["optional", "excluded"].includes(comp.scope)
     ) {