@cdxgen/cdxgen 13.5.0 → 13.7.0

package/lib/inventory/evidenceUtils.js

@@ -6,6 +6,7 @@ import { build } from "@cdxgen/cdx-purl";
 import { safeExistsSync } from "../core/fs.js";
 import { thoughtLog } from "../core/logger.js";
 import { sanitizeBomUrl } from "../core/propertySanitizer.js";
+import { vscodeExtensionPlatformFromPath } from "../parsers/vscodeExtensionId.js";
 import { isAgentSkillSpecValid } from "./agentSkillSpec.js";
 import {
   addDosaiReachabilityProperties,
@@ -22,6 +23,7 @@ import {
 } from "./dosaiParsers.js";
 import { createGtfoBinsPropertiesFromRow } from "./gtfobins.js";
 import { createLolbasProperties } from "./lolbas.js";
+import { occurrenceLocationKey } from "./occurrenceKey.js";
 import { isOsPackagePurlType } from "./osPackagePurl.js";
 import {
   createOsQueryFallbackBomRef,
@@ -369,7 +371,7 @@ export function addEvidenceForDotnet(pkgList, slicesFile, options = {}) {
           addDosaiSetValue(
             purlLocationMap,
             modPurl,
-            `${adep.Path}#${adep.LineNumber}`,
+            occurrenceLocationKey(adep.Path, adep.LineNumber),
           );
         }
       }
@@ -619,6 +621,13 @@ export function convertOSQueryResults(
         // The profile an add-on was found in is not part of its purl, so the
         // add-on ID is what tells apart two add-ons that share a name.
         qualifiers = { tag_id: firefoxAddonTagId(res.identifier) };
+      } else if (queryObj.purlType === "vscode-extension") {
+        // A platform-specific build is a different artefact from the one for
+        // another platform, and the extension scan names it the same way.
+        const targetPlatform = vscodeExtensionPlatformFromPath(res.path);
+        if (targetPlatform) {
+          qualifiers = { platform: targetPlatform };
+        }
       }
       if (name) {
         name = sanitizeOsQueryIdentity(name);
@@ -788,11 +797,14 @@ export function convertOSQueryResults(
 // profile. Where it was found is not part of what it is, so the path is an
 // occurrence rather than a subpath: a home directory stays out of the purl, and
 // one package installed in two places is one component with two occurrences.
+// Chocolatey rows carry the absolute path of the package's .nuspec, which a
+// purl subpath cannot hold, so it is kept as an occurrence like these.
 const LOCATED_PACKAGE_PURL_TYPES = new Set([
   "npm",
   "pypi",
   "chrome-extension",
   "vscode-extension",
+  "chocolatey",
 ]);
 const LOCATED_PACKAGE_CATEGORIES = new Set(["firefox_addons"]);
 

package/lib/inventory/fetchBatch.js

@@ -221,7 +221,7 @@ export function openHostCircuit(
     );
   } else if (breaker === "maven-central") {
     console.warn(
-      `${host} ${what}, so cdxgen will not contact it for the next ${minutes} minute(s) and Maven metadata comes from local caches only. Set MAVEN_CENTRAL_URL to a mirror or repository manager, for example https://maven-central.storage-download.googleapis.com/maven2/, to keep license enrichment working.`,
+      `${host} ${what}, so cdxgen will not contact it for the next ${minutes} minute(s) and Maven metadata comes from local caches only. Set MAVEN_CENTRAL_MIRROR=true to use Google's Maven Central mirror, or name a repository manager in MAVEN_CENTRAL_URL, to keep license enrichment working.`,
     );
   } else {
     console.warn(

package/lib/inventory/formulationParsers.js

@@ -4,7 +4,9 @@ import { basename } from "node:path";
 import process from "node:process";
 
 import { readEnvironmentVariable } from "../core/activity.js";
+import { PROJECT_TYPE_ALIASES } from "../core/env.js";
 import { getAllFiles } from "../core/fs.js";
+import { thoughtLog } from "../core/logger.js";
 import { collectOSCryptoLibs } from "./cbomutils.js";
 import { azurePipelinesParser } from "./ciParsers/azurePipelines.js";
 import { circleCiParser } from "./ciParsers/circleCi.js";
@@ -12,6 +14,13 @@ import { githubActionsParser } from "./ciParsers/githubActions.js";
 import { gitlabCiParser } from "./ciParsers/gitlabCi.js";
 import { jenkinsParser } from "./ciParsers/jenkins.js";
 import { trimComponents } from "./depsUtils.js";
+import {
+  dotnetProjectFormulationParser,
+  formulationSourcePath,
+  goModuleFormulationParser,
+  makefileFormulationParser,
+  pythonRequirementsFormulationParser,
+} from "./ecosystemFormulationParsers.js";
 import {
   collectEnvInfo,
   getBranch,
@@ -47,8 +56,9 @@ function buildReadmeSecurityComponents(discoveryPath, options) {
     if (!scan.hasHiddenUnicode) {
       continue;
     }
+    const sourcePath = formulationSourcePath(filePath, discoveryPath);
     const properties = [
-      { name: "internal:SrcFile", value: filePath },
+      { name: "internal:SrcFile", value: sourcePath },
       { name: "cdx:file:kind", value: "readme" },
       { name: "cdx:file:hasHiddenUnicode", value: "true" },
       {
@@ -71,7 +81,7 @@ function buildReadmeSecurityComponents(discoveryPath, options) {
       });
     }
     components.push({
-      "bom-ref": `file:${filePath}`,
+      "bom-ref": `file:${sourcePath}`,
       name: basename(filePath),
       properties,
       type: "file",
@@ -90,13 +100,22 @@ function buildReadmeSecurityComponents(discoveryPath, options) {
  * {
  *   id:       string,                         // unique stable identifier
  *   patterns: string[],                       // non-empty array of glob patterns for file discovery
- *   parse(files: string[], options: Object):  // synchronous function
+ *   parse(files: string[], options: Object, context: { rootDir: string }):
+ *                                             // synchronous function
  *     { workflows?, components?, services?, properties?, dependencies? }
  * }
  * ```
+ *
+ * `context.rootDir` is the scanned project root. A parser that records file
+ * paths records them relative to it, so no path of the scanning machine
+ * reaches the BOM.
  */
 const _parsers = [
   rustFormulationParser,
+  makefileFormulationParser,
+  goModuleFormulationParser,
+  pythonRequirementsFormulationParser,
+  dotnetProjectFormulationParser,
   githubActionsParser,
   gitlabCiParser,
   jenkinsParser,
@@ -187,6 +206,59 @@ const ENV_BLOCKLIST = [
   "webhook",
 ];
 
+/**
+ * Remove the parts of every URL in an environment value that can carry
+ * credentials: the userinfo before `@`, the query string and the fragment.
+ * The value is scanned with indexOf, one URL at a time, so the work stays
+ * linear in its length.
+ *
+ * @param {string} value environment variable value
+ * @returns {string} the value with URL credentials, queries and fragments removed
+ */
+function stripUrlCredentials(value) {
+  const delimiters = " \t\r\n\"'`,;<>|";
+  let result = "";
+  let cursor = 0;
+  let schemeEnd = value.indexOf("://");
+  while (schemeEnd !== -1) {
+    const start = schemeEnd + 3;
+    let end = start;
+    while (end < value.length && !delimiters.includes(value[end])) {
+      end += 1;
+    }
+    const url = value.slice(start, end);
+    let authorityEnd = url.length;
+    for (const marker of ["/", "?", "#"]) {
+      const at = url.indexOf(marker);
+      if (at !== -1 && at < authorityEnd) {
+        authorityEnd = at;
+      }
+    }
+    const authority = url.slice(0, authorityEnd);
+    let rest = url.slice(authorityEnd);
+    for (const marker of ["?", "#"]) {
+      const at = rest.indexOf(marker);
+      if (at !== -1) {
+        rest = rest.slice(0, at);
+      }
+    }
+    result += `${value.slice(cursor, start)}${authority.slice(authority.lastIndexOf("@") + 1)}${rest}`;
+    cursor = end;
+    schemeEnd = value.indexOf("://", end);
+  }
+  return result + value.slice(cursor);
+}
+
+// The project types a scan asked for, as given on the command line.
+function requestedProjectTypes(options) {
+  return [].concat(options?.projectType || []).flatMap((type) =>
+    `${type}`
+      .split(",")
+      .map((part) => part.trim().toLowerCase())
+      .filter(Boolean),
+  );
+}
+
 /**
  * Build the formulation section for a CycloneDX BOM.
  *
@@ -285,15 +357,45 @@ export function addFormulationSection(filePath, options, context = {}) {
     }
   }
 
+  // The toolchains and OS crypto libraries are probed on the machine cdxgen
+  // runs on. A container image or root filesystem was built elsewhere, so the
+  // scanning host's tools say nothing about it.
+  // Only an explicit container type counts: hasAnyProjectType treats
+  // `universal` as every type, and a universal source scan runs on the host.
+  const requestedTypes = requestedProjectTypes(options);
+  const describesScanningHost = !requestedTypes.some((type) =>
+    PROJECT_TYPE_ALIASES.oci.includes(type),
+  );
+  // An OS inventory (obom) describes the live host. Its path is usually the
+  // working directory, such as a home directory or /, so the build-input,
+  // CI and README discovery below would walk a tree that is not a project.
+  const describesLiveHost =
+    requestedTypes.length > 0 &&
+    requestedTypes.every((type) => PROJECT_TYPE_ALIASES.os.includes(type));
+  if (!describesScanningHost) {
+    thoughtLog(
+      "Skipping the toolchain and OS crypto library probes for formulation, since they would describe the scanning host and not the image.",
+    );
+  }
+
   // Build environment details (Java, .NET, Python, Node, GCC, Rust, Go, Ruby,
-  // PHP, Composer)
-  const infoComponents = collectEnvInfo(projectPath, options);
+  // RubyGems, Bundler, PHP, Composer)
+  const infoComponents = describesScanningHost
+    ? collectEnvInfo(projectPath, options)
+    : [];
   if (infoComponents?.length) {
     components = components.concat(infoComponents);
   }
 
-  // OS crypto libraries (cbom mode)
-  if (options.includeCrypto) {
+  // OS crypto libraries (cbom mode). The queries also read the host's
+  // certificates, authorized keys and keychain items, so they run only when
+  // formulation was requested explicitly, not when CI, a profile, an OS type
+  // or the cbom command turned it on.
+  if (
+    options.includeCrypto &&
+    describesScanningHost &&
+    !options.formulationAutomatic
+  ) {
     const cryptoLibs = collectOSCryptoLibs(options, context?.executeOsQuery);
     if (cryptoLibs?.length) {
       components = components.concat(cryptoLibs);
@@ -307,8 +409,13 @@ export function addFormulationSection(filePath, options, context = {}) {
   const ciProperties = [];
 
   const discoveryPath = projectPath || ".";
+  if (describesLiveHost) {
+    thoughtLog(
+      "Skipping build-input, CI and README discovery for formulation, since an OS inventory describes the host and not a project tree.",
+    );
+  }
 
-  for (const parser of _parsers) {
+  for (const parser of describesLiveHost ? [] : _parsers) {
     const matchedFiles = [];
     for (const pattern of parser.patterns) {
       const found = getAllFiles(discoveryPath, pattern, options);
@@ -323,7 +430,9 @@ export function addFormulationSection(filePath, options, context = {}) {
 
     let result;
     try {
-      result = parser.parse(uniqueMatchedFiles, options);
+      result = parser.parse(uniqueMatchedFiles, options, {
+        rootDir: discoveryPath,
+      });
     } catch (err) {
       // A broken parser must not kill SBOM generation
       console.warn(
@@ -355,10 +464,9 @@ export function addFormulationSection(filePath, options, context = {}) {
     components = components.concat(ciComponents);
   }
 
-  const readmeSecurityComponents = buildReadmeSecurityComponents(
-    discoveryPath,
-    options,
-  );
+  const readmeSecurityComponents = describesLiveHost
+    ? []
+    : buildReadmeSecurityComponents(discoveryPath, options);
   if (readmeSecurityComponents.length) {
     components = components.concat(readmeSecurityComponents);
   }
@@ -377,7 +485,9 @@ export function addFormulationSection(filePath, options, context = {}) {
       !ENV_BLOCKLIST.some((b) => value.toLowerCase().includes(b)) &&
       value.length
     ) {
-      environmentVars.push({ name: aevar, value });
+      // A registry or mirror URL can carry a password before `@` or a
+      // token in its query string, so only the scheme, host and path stay.
+      environmentVars.push({ name: aevar, value: stripUrlCredentials(value) });
     }
   }
 

package/lib/inventory/gtfobins.js

@@ -17,6 +17,15 @@ const MATCH_FIELDS = [
   "executable",
   "description",
 ];
+// Rows of these categories name an account, not a program: a user called
+// `node` or `git` is no use of the binary of that name, and the GECOS comment
+// in the description is free text about the account.
+const ACCOUNT_CATEGORIES = new Set([
+  "authorized_keys_snapshot",
+  "logged_in_users_snapshot",
+  "users_snapshot",
+]);
+const ACCOUNT_SKIPPED_FIELDS = new Set(["name", "description"]);
 const UNIX_PATH_PATTERN = /(?:^|[\s'"`=;|&(])((?:\.{0,2}\/|\/)[^\s'"`|;&()]+)/g;
 const STANDALONE_COMMAND_PATTERN =
   /(?:^|[\s;|&(])([a-z_][a-z0-9+._-]{1,})(?=$|[\s'"`|;&()])/gi;
@@ -250,7 +259,11 @@ export function createGtfoBinsProperties(name, linkedName) {
  */
 export function createGtfoBinsPropertiesFromRow(queryCategory, row) {
   const matches = new Map();
+  const isAccount = ACCOUNT_CATEGORIES.has(queryCategory);
   for (const field of MATCH_FIELDS) {
+    if (isAccount && ACCOUNT_SKIPPED_FIELDS.has(field)) {
+      continue;
+    }
     const fieldValue = row?.[field];
     if (!fieldValue) {
       continue;

package/lib/inventory/hbom.js

@@ -8,6 +8,9 @@ import {
   readEnvironmentVariable,
   recordActivity,
 } from "../core/activity.js";
+import { thoughtLog } from "../core/logger.js";
+import { isCycloneDx20SpecVersion } from "./bomUtils.js";
+import { readCycloneDx20 } from "./cyclonedx20.js";
 import { getHbomCommandDiagnosticSummary } from "./hbomAnalysis.js";
 import { importHbomModule } from "./hbomLoader.js";
 import { resolveCdxgenPlugins, resolvePluginBinary } from "./plugins.js";
@@ -411,8 +414,9 @@ export function ensureSupportedHbomSpecVersion(specVersion) {
   if (specVersion === undefined || specVersion === null || specVersion === "") {
     return;
   }
-  if (Number(specVersion) !== 1.7) {
-    throw new Error("HBOM generation currently supports only CycloneDX 1.7.");
+  // cdxgen renders 2.0 itself from the 1.7 collector output.
+  if (Number(specVersion) !== 1.7 && Number(specVersion) !== 2) {
+    throw new Error("HBOM generation supports only CycloneDX 1.7 and 2.0.");
   }
 }
 
@@ -474,6 +478,10 @@ export function normalizeHbomOptions(options = {}) {
     includeSensitiveIdentifiers:
       options.includeSensitiveIdentifiers ?? options.sensitive ?? false,
     platform: options.platform,
+    // Always 1.7: cdxgen's own pipeline (postProcess and its 2.0 renderer)
+    // turns the 1.7 collector output into 2.0, the way it renders every
+    // other BOM. A cdx-hbom that can emit 2.0 itself is never asked to.
+    specVersion: "1.7",
     timeoutMs:
       Number.isNaN(timeoutMs) || timeoutMs <= 0 ? undefined : timeoutMs,
   };
@@ -722,6 +730,38 @@ export function addHbomAnalysisProperties(bomJson) {
   return bomJson;
 }
 
+/**
+ * The 1.x-shaped model of a collector result. cdxgen asks the collector for
+ * 1.7; a collector that still answers with a CycloneDX 2.0 document (a
+ * `specFormat`, or specVersion 2.0) is read with readCycloneDx20, so the
+ * 1.x pipeline that follows never mistakes it for a 1.x BOM and loses its
+ * inventories.
+ *
+ * @param {object} collected What collectHardware returned
+ * @returns {object} The 1.x-shaped HBOM
+ */
+export function toInternalHbom(collected) {
+  if (
+    !collected ||
+    typeof collected !== "object" ||
+    !(collected.specFormat || isCycloneDx20SpecVersion(collected.specVersion))
+  ) {
+    return collected;
+  }
+  const { internalBom, losses } = readCycloneDx20(collected);
+  if (losses.length) {
+    thoughtLog(
+      `The HBOM collector returned CycloneDX 2.0; reading it into the 1.x model reported ${losses.length} loss(es).`,
+      losses,
+    );
+  }
+  // The pipeline renders 2.0 again from the 1.x model, so the model says
+  // what the collector was asked for.
+  internalBom.specVersion = "1.7";
+  delete internalBom.specFormat;
+  return internalBom;
+}
+
 /**
  * Generate an HBOM using the optional cdx-hbom package.
  *
@@ -750,12 +790,11 @@ export async function createHbomDocument(options = {}) {
     });
     return createDryRunHbomDocument(options);
   }
-  const bomJson = addHbomAnalysisProperties(
-    await hbomModule.collectHardware(normalizedOptions),
-  );
+  const collected = await hbomModule.collectHardware(normalizedOptions);
+  const bomJson = addHbomAnalysisProperties(toInternalHbom(collected));
   if (isDryRun) {
     recordHbomCollectorTrace(
-      hbomModule.getCollectorTrace?.(bomJson) ?? normalizedOptions.trace,
+      hbomModule.getCollectorTrace?.(collected) ?? normalizedOptions.trace,
     );
   }
   return bomJson;

package/lib/inventory/hbomLoader.js

@@ -7,6 +7,21 @@ const LOCAL_HBOM_MODULE_URL = new URL(
   import.meta.url,
 );
 const LOCAL_HBOM_MODULE_PATH = fileURLToPath(LOCAL_HBOM_MODULE_URL);
+// The ./cdx20 export of @cdxgen/cdx-hbom (cdx-hbom#26), at the same place
+// relative to a local checkout as the package's exports map gives it.
+const LOCAL_HBOM_MAPPER_PATH = fileURLToPath(
+  new URL("../../cdx-hbom/src/common/cdx20.js", import.meta.url),
+);
+
+/**
+ * The places cdx-hbom is loaded from, in order: a local checkout next to
+ * cdxgen, else the installed package.
+ */
+export const HBOM_MODULE_LOCATIONS = Object.freeze({
+  localModulePath: LOCAL_HBOM_MODULE_PATH,
+  localMapperPath: LOCAL_HBOM_MAPPER_PATH,
+  packageMapperSpecifier: "@cdxgen/cdx-hbom/cdx20",
+});
 
 /**
  * Resolve the optional cdx-hbom module.
@@ -33,3 +48,154 @@ export async function importHbomModule() {
   }
   return hbomModule;
 }
+
+/**
+ * Load the optional CycloneDX 2.0 hardware mapper,
+ * `toCdx20HardwareComponent`, from the same cdx-hbom that importHbomModule
+ * loads: the local checkout when it exists (its src/common/cdx20.js), else
+ * the installed package's `./cdx20` export. A cdx-hbom without the mapper
+ * (the published 0.7.0 has no `./cdx20` export) or one that fails to load
+ * gives null, and the 2.0 render then keeps its generic conversion.
+ *
+ * @param {object} [locations] Where to look; defaults to
+ *   HBOM_MODULE_LOCATIONS
+ * @returns {Promise<Function|null>} The mapper, or null
+ */
+export async function loadHardwareMapper(locations = HBOM_MODULE_LOCATIONS) {
+  let specifier = locations.packageMapperSpecifier;
+  if (safeExistsSync(locations.localModulePath)) {
+    if (!safeExistsSync(locations.localMapperPath)) {
+      return null;
+    }
+    specifier = pathToFileURL(locations.localMapperPath).href;
+  }
+  let mapperModule;
+  try {
+    mapperModule = await import(specifier);
+  } catch (err) {
+    // cdx-hbom not installed, or a release without the ./cdx20 export: the
+    // generic conversion applies and nothing is wrong. Any other failure is
+    // a broken cdx-hbom and is reported before falling back.
+    if (!isMissingMapperError(err, specifier)) {
+      console.warn(
+        `The cdx-hbom CycloneDX 2.0 hardware mapper could not be loaded, so hardware components use the generic 2.0 conversion: ${err?.message || err}`,
+      );
+    }
+    return null;
+  }
+  return typeof mapperModule.toCdx20HardwareComponent === "function"
+    ? mapperModule.toCdx20HardwareComponent
+    : null;
+}
+
+/**
+ * Whether an import error means the mapper is simply absent: the package is
+ * not installed, or its exports map has no `./cdx20`. A missing module
+ * imported from inside the mapper is a fault, not an absence.
+ *
+ * @param {Error} err Import error
+ * @param {string} specifier The specifier that was imported
+ * @returns {boolean} true when the mapper is absent
+ */
+function isMissingMapperError(err, specifier) {
+  if (err?.code === "ERR_PACKAGE_PATH_NOT_EXPORTED") {
+    return true;
+  }
+  const message = String(err?.message || "");
+  const packageName = specifier.startsWith("@")
+    ? specifier.split("/").slice(0, 2).join("/")
+    : undefined;
+  if (packageName) {
+    // Node.js and Bun name the missing package; Deno reports an import that
+    // its import map or node_modules cannot resolve.
+    return (
+      message.includes(`'${packageName}'`) ||
+      message.includes(`"${packageName}"`) ||
+      message.includes(`${packageName}/cdx20`) ||
+      (err?.code === "ERR_MODULE_NOT_FOUND" && message.includes(packageName))
+    );
+  }
+  return false;
+}
+
+/** The identifier policy cdx-hbom records when raw identifiers were kept. */
+export const RAW_IDENTIFIER_POLICY = "raw-identifiers-enabled";
+
+/**
+ * The identifier policy the collector recorded for the run, on the HBOM
+ * subject (`metadata.component`) or in `metadata.properties`.
+ *
+ * @param {object} internalBom The 1.x-shaped internal model
+ * @returns {string|undefined} The `cdx:hbom:identifierPolicy` value
+ */
+export function recordedIdentifierPolicy(internalBom) {
+  for (const properties of [
+    internalBom?.metadata?.component?.properties,
+    internalBom?.metadata?.properties,
+  ]) {
+    const value = (Array.isArray(properties) ? properties : []).find(
+      (property) => property?.name === "cdx:hbom:identifierPolicy",
+    )?.value;
+    if (typeof value === "string") {
+      return value;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * The renderCycloneDx20 context for an internal model: the hardware mapper
+ * and its identifier options when the model has a component with a
+ * cdx:hbom:* property and a mapper is available, otherwise an empty context,
+ * so a BOM without hardware never loads cdx-hbom. Hardware identifiers are
+ * asserted only when the collector recorded `raw-identifiers-enabled` for the
+ * run (it does when HBOM generation was asked to include sensitive
+ * identifiers); a redacted run never yields identifiers, whichever command
+ * renders it.
+ *
+ * @param {object} internalBom The 1.x-shaped internal model
+ * @param {object} [locations] Passed to loadHardwareMapper
+ * @returns {Promise<{hardwareMapper?: Function, hardwareOptions?: {includeSensitiveIdentifiers: boolean, identifierPolicy?: string}}>}
+ *   The render context
+ */
+export async function hardwareRenderContext(internalBom, locations) {
+  if (!hasHardwareComponent(internalBom)) {
+    return {};
+  }
+  const hardwareMapper = await loadHardwareMapper(locations);
+  if (!hardwareMapper) {
+    return {};
+  }
+  const identifierPolicy = recordedIdentifierPolicy(internalBom);
+  const hardwareOptions = {
+    includeSensitiveIdentifiers: identifierPolicy === RAW_IDENTIFIER_POLICY,
+  };
+  if (identifierPolicy !== undefined) {
+    hardwareOptions.identifierPolicy = identifierPolicy;
+  }
+  return { hardwareMapper, hardwareOptions };
+}
+
+function hasHardwareComponent(internalBom) {
+  const pending = [
+    internalBom?.metadata?.component,
+    ...(Array.isArray(internalBom?.components) ? internalBom.components : []),
+  ];
+  while (pending.length) {
+    const component = pending.pop();
+    if (!component || typeof component !== "object") {
+      continue;
+    }
+    if (
+      (component.properties || []).some((property) =>
+        `${property?.name ?? ""}`.startsWith("cdx:hbom:"),
+      )
+    ) {
+      return true;
+    }
+    if (Array.isArray(component.components)) {
+      pending.push(...component.components);
+    }
+  }
+  return false;
+}