@cdxgen/cdxgen 13.4.0 → 13.6.0

package/lib/inventory/evidenceUtils.js

@@ -1,10 +1,13 @@
 import { platform } from "node:os";
-import { extname } from "node:path";
+import { basename, dirname, extname } from "node:path";
 
 import { build } from "@cdxgen/cdx-purl";
 
 import { safeExistsSync } from "../core/fs.js";
 import { thoughtLog } from "../core/logger.js";
+import { sanitizeBomUrl } from "../core/propertySanitizer.js";
+import { vscodeExtensionPlatformFromPath } from "../parsers/vscodeExtensionId.js";
+import { isAgentSkillSpecValid } from "./agentSkillSpec.js";
 import {
   addDosaiReachabilityProperties,
   readDosaiMethodsReport,
@@ -20,16 +23,26 @@ import {
 } from "./dosaiParsers.js";
 import { createGtfoBinsPropertiesFromRow } from "./gtfobins.js";
 import { createLolbasProperties } from "./lolbas.js";
+import { isOsPackagePurlType } from "./osPackagePurl.js";
 import {
   createOsQueryFallbackBomRef,
+  createOsQueryPackageIdentity,
   createOsQueryPurl,
   deriveOsQueryDescription,
   deriveOsQueryName,
   deriveOsQueryPublisher,
   deriveOsQueryVersion,
+  OS_PACKAGE_LICENSE_COLUMNS,
+  osPackagePropertyName,
   sanitizeOsQueryIdentity,
   shouldCreateOsQueryPurl,
 } from "./osqueryTransform.js";
+import {
+  adjustLicenseInformation,
+  findLicenseId,
+  isSpdxLicenseExpression,
+  spdxLicenses,
+} from "./spdx.js";
 
 /**
  * Build a CycloneDX occurrence evidence object from a location string.
@@ -540,20 +553,55 @@ export function convertOSQueryResults(
   enhance = false,
   osPackageListers = undefined,
 ) {
+  if (queryCategory === "ai_agent_skills") {
+    return convertAgentSkillRows(results);
+  }
   const pkgList = [];
+  // deb, rpm, apk, pacman, Homebrew, MacPorts, snap and flatpak rows describe a
+  // package, not a file or an app: the package manager names them, and nothing
+  // in such a row is a path inside the package.
+  const isOsPackage = isOsPackagePurlType(queryObj.purlType);
+  const isLocatedPackage =
+    LOCATED_PACKAGE_PURL_TYPES.has(queryObj.purlType) ||
+    LOCATED_PACKAGE_CATEGORIES.has(queryCategory);
   if (results?.length) {
     for (const res of results) {
-      const version = deriveOsQueryVersion(res);
+      let version = deriveOsQueryVersion(res);
       let name = deriveOsQueryName(res, results.length === 1, queryObj.name);
       if (queryObj.purlType === "chrome-extension") {
-        name = (res.identifier || res.extension_id || name || "").toLowerCase();
+        // osquery computes identifier from the manifest key, which an
+        // unpacked extension may lack; the profile still knows its ID.
+        name = (
+          res.identifier ||
+          res.referenced_identifier ||
+          res.extension_id ||
+          name ||
+          ""
+        ).toLowerCase();
+      }
+      let npmScope;
+      if (queryObj.purlType === "npm") {
+        const scoped = /^@([^/]+)\/(.+)$/.exec(`${name || ""}`);
+        if (scoped) {
+          [, npmScope, name] = scoped;
+        }
       }
       let group = "";
-      const subpath = res.path || res.admindir || res.source;
+      const subpath =
+        isOsPackage || isLocatedPackage
+          ? undefined
+          : res.path || res.admindir || res.source;
       const publisher = deriveOsQueryPublisher(res);
-      // For vscode-extension purl type, the publisher is used as the namespace
-      if (queryObj.purlType === "vscode-extension" && publisher) {
-        group = publisher.toLowerCase();
+      if (queryObj.purlType === "vscode-extension") {
+        // osquery names a VS Code extension by its ID, <publisher>.<name>,
+        // which the vscode-extension type splits into namespace and name. Its
+        // publisher column is the publisher's display name.
+        const extensionId = /^([^.]+)\.(.+)$/u.exec(`${name || ""}`);
+        if (extensionId) {
+          [, group, name] = extensionId;
+        } else if (publisher) {
+          group = publisher.toLowerCase();
+        }
       }
       let scope;
       const compScope = res.priority;
@@ -568,25 +616,60 @@ export function convertOSQueryResults(
         qualifiers = {
           tag_id: res.identifying_number.replace("{", "").replace("}", ""),
         };
+      } else if (queryCategory === "firefox_addons" && res.identifier) {
+        // The profile an add-on was found in is not part of its purl, so the
+        // add-on ID is what tells apart two add-ons that share a name.
+        qualifiers = { tag_id: firefoxAddonTagId(res.identifier) };
+      } else if (queryObj.purlType === "vscode-extension") {
+        // A platform-specific build is a different artefact from the one for
+        // another platform, and the extension scan names it the same way.
+        const targetPlatform = vscodeExtensionPlatformFromPath(res.path);
+        if (targetPlatform) {
+          qualifiers = { platform: targetPlatform };
+        }
       }
       if (name) {
         name = sanitizeOsQueryIdentity(name);
-        group = sanitizeOsQueryIdentity(group);
+        // sanitizeOsQueryIdentity drops a leading @, which an npm scope keeps.
+        group = npmScope
+          ? `@${sanitizeOsQueryIdentity(npmScope)}`
+          : sanitizeOsQueryIdentity(group);
         const isCryptoAsset = queryObj.componentType === "cryptographic-asset";
-        const purl = shouldCreateOsQueryPurl(queryObj.componentType)
-          ? createOsQueryPurl(
-              queryObj.purlType,
-              group,
-              name,
-              version,
-              qualifiers,
-              subpath,
-            )
-          : undefined;
+        let purl;
+        if (isOsPackage) {
+          // The version of such a row is part of its identity: an rpm's is its
+          // version and release, behind its epoch.
+          const identity = createOsQueryPackageIdentity(
+            queryObj.purlType,
+            res,
+            name,
+          );
+          version = identity.version;
+          purl = shouldCreateOsQueryPurl(queryObj.componentType)
+            ? identity.purl
+            : undefined;
+        } else if (shouldCreateOsQueryPurl(queryObj.componentType)) {
+          purl = createOsQueryPurl(
+            queryObj.purlType,
+            group,
+            name,
+            version,
+            qualifiers,
+            subpath,
+          );
+        }
         const props = [{ name: "cdx:osquery:category", value: queryCategory }];
         props.push(...createLolbasProperties(queryCategory, res));
         if (platform() === "linux") {
-          props.push(...createGtfoBinsPropertiesFromRow(queryCategory, res));
+          // The GTFOBins matcher reads a description for command names, and
+          // the free text of a package's description matches by chance: zlib's
+          // "the deflate compression method found in gzip" is no use of gzip.
+          props.push(
+            ...createGtfoBinsPropertiesFromRow(
+              queryCategory,
+              isOsPackage ? { ...res, description: undefined } : res,
+            ),
+          );
         }
         let providesList;
         if (enhance) {
@@ -620,10 +703,12 @@ export function convertOSQueryResults(
           version: version || "",
           description,
           publisher,
+          // Without a purl, the bom-ref is built from the row, and the path of
+          // a located package is not part of what it is.
           "bom-ref": createOsQueryBomRef(
             queryCategory,
             queryObj.componentType,
-            res,
+            isLocatedPackage ? { ...res, path: undefined } : res,
             name,
             version,
             purl,
@@ -644,25 +729,364 @@ export function convertOSQueryResults(
         if (cryptoProperties) {
           apkg.cryptoProperties = cryptoProperties;
         }
+        const licenseColumns = isOsPackage
+          ? OS_PACKAGE_LICENSE_COLUMNS[queryObj.purlType]
+          : undefined;
+        if (licenseColumns) {
+          const licenses = osPackageLicenses(
+            res[licenseColumns.license],
+            licenseColumns.separator,
+          );
+          if (licenses.length) {
+            apkg.licenses = licenses;
+          }
+          const website = licenseColumns.website
+            ? osPackageWebsite(res[licenseColumns.website])
+            : undefined;
+          if (website) {
+            apkg.externalReferences = [{ type: "website", url: website }];
+          }
+        }
         for (const k of Object.keys(res).filter((p) => {
           if (["version", "description", "publisher"].includes(p)) {
             return false;
           }
           return !(queryObj.purlType !== "chrome-extension" && p === "name");
         })) {
-          if (res[k] && res[k] !== "null") {
+          const propertyName = isOsPackage
+            ? osPackagePropertyName(queryObj.purlType, k)
+            : k;
+          // (none) is what rpm's database says for a tag a record lacks, such
+          // as the source rpm of a key record.
+          if (
+            propertyName &&
+            res[k] &&
+            res[k] !== "null" &&
+            !(isOsPackage && res[k] === "(none)")
+          ) {
             props.push({
-              name: k,
+              name: propertyName,
               value: res[k],
             });
           }
         }
+        if (isLocatedPackage) {
+          // osquery writes a JSON null as "null", which is the path Firefox
+          // gives its default theme.
+          const occurrence = createOccurrenceEvidence(
+            [res.path, res.directory].find(
+              (value) => value && value !== "null",
+            ),
+          );
+          if (occurrence) {
+            apkg.evidence = { occurrences: [occurrence] };
+          }
+        }
         apkg.properties = props;
         pkgList.push(apkg);
       }
     }
   }
-  return pkgList;
+  return isLocatedPackage ? mergeLocatedPackages(pkgList) : pkgList;
+}
+
+// npm and pypi rows name a package found at a path on the host, Chromium and
+// VS Code extension rows an extension found in a browser profile or an editor's
+// extensions folder, and a firefox_addons row an add-on found in a Firefox
+// profile. Where it was found is not part of what it is, so the path is an
+// occurrence rather than a subpath: a home directory stays out of the purl, and
+// one package installed in two places is one component with two occurrences.
+// Chocolatey rows carry the absolute path of the package's .nuspec, which a
+// purl subpath cannot hold, so it is kept as an occurrence like these.
+const LOCATED_PACKAGE_PURL_TYPES = new Set([
+  "npm",
+  "pypi",
+  "chrome-extension",
+  "vscode-extension",
+  "chocolatey",
+]);
+const LOCATED_PACKAGE_CATEGORIES = new Set(["firefox_addons"]);
+
+const FIREFOX_ADDON_GUID =
+  /^\{([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\}$/iu;
+
+// A Firefox add-on ID is an email-like string or a GUID in braces. A swid
+// tag_id that is a GUID must be in lower case, and is written without braces.
+function firefoxAddonTagId(identifier) {
+  const id = `${identifier}`.trim();
+  const guid = FIREFOX_ADDON_GUID.exec(id);
+  return guid ? guid[1].toLowerCase() : id;
+}
+
+function mergeLocatedPackages(components) {
+  const byRef = new Map();
+  for (const component of components) {
+    const existing = byRef.get(component["bom-ref"]);
+    if (!existing) {
+      byRef.set(component["bom-ref"], component);
+      continue;
+    }
+    const occurrences = existing.evidence?.occurrences || [];
+    for (const occurrence of component.evidence?.occurrences || []) {
+      if (
+        !occurrences.some((known) => known.location === occurrence.location)
+      ) {
+        occurrences.push(occurrence);
+      }
+    }
+    if (occurrences.length) {
+      existing.evidence = { ...existing.evidence, occurrences };
+    }
+    for (const property of component.properties) {
+      if (
+        !existing.properties.some(
+          (known) =>
+            known.name === property.name && known.value === property.value,
+        )
+      ) {
+        existing.properties.push(property);
+      }
+    }
+  }
+  for (const component of byRef.values()) {
+    component.evidence?.occurrences?.sort((a, b) =>
+      a.location.localeCompare(b.location),
+    );
+  }
+  return [...byRef.values()];
+}
+
+// The licence an OS package declares. An apk `L:` field is a list of SPDX ids
+// separated by spaces on older packages and an SPDX expression on newer ones;
+// pacman keeps one licence per entry, each an SPDX id or expression, which
+// osquery joins with commas; a snap or a flatpak declares one expression. Only
+// an explicit operator or a parenthesis makes a text an expression: the SPDX
+// parser's own check takes the `or` inside `LGPL-2.0-or-later` for an operator,
+// and would turn a plain list into one expression. A token is an id only when
+// the SPDX list has what it maps to; any other is kept by name rather than
+// dropped.
+const MAX_OS_PACKAGE_LICENSES = 64;
+const SPDX_OPERATORS = new Set(["AND", "OR", "WITH"]);
+
+function isExplicitSpdxExpression(text) {
+  // No SPDX expression holds a comma, but the expression parser lets one
+  // through, which would take a whole pacman licence list for one expression.
+  return (
+    !text.includes(",") &&
+    (text.split(/\s+/u).some((token) => SPDX_OPERATORS.has(token)) ||
+      text.includes("(")) &&
+    isSpdxLicenseExpression(text)
+  );
+}
+
+function osPackageLicenses(text, separator) {
+  const license = `${text ?? ""}`.split(/\s+/u).filter(Boolean).join(" ");
+  if (!license) {
+    return [];
+  }
+  if (isExplicitSpdxExpression(license)) {
+    return [{ expression: license }];
+  }
+  const entries = [];
+  const items = separator ? license.split(separator) : [license];
+  for (const item of items.slice(0, MAX_OS_PACKAGE_LICENSES)) {
+    const entry = item.trim();
+    if (!entry || SPDX_OPERATORS.has(entry)) {
+      continue;
+    }
+    if (isExplicitSpdxExpression(entry)) {
+      entries.push({ expression: entry });
+      continue;
+    }
+    const id = findLicenseId(entry);
+    entries.push(
+      spdxLicenses.includes(id)
+        ? { license: { id } }
+        : { license: { name: entry } },
+    );
+  }
+  // A list that holds an expression must become one expression, since a
+  // component's licenses are either licences or a single expression. Every
+  // entry applies, so they are joined with AND.
+  if (entries.length > 1 && entries.some((entry) => entry.expression)) {
+    const parts = entries.map((entry) =>
+      entry.expression ? `(${entry.expression})` : entry.license.id,
+    );
+    const expression = parts.join(" AND ");
+    if (parts.every(Boolean) && isSpdxLicenseExpression(expression)) {
+      return [{ expression }];
+    }
+  }
+  return adjustLicenseInformation(entries);
+}
+
+// The home page of a package. A URL a package carries is untrusted text: only
+// an http(s) URL is kept, without credentials, query or fragment.
+function osPackageWebsite(value) {
+  const cleaned = sanitizeBomUrl(`${value ?? ""}`.trim());
+  try {
+    const { protocol } = new URL(cleaned);
+    return protocol === "http:" || protocol === "https:" ? cleaned : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Convert the packages of an apk installed database into components.
+ *
+ * The packages are named as the container scan names an Alpine, Wolfi or
+ * Chainguard package, and grouped with the other OS package categories under
+ * `cdx:osquery:category` = `apk_packages`. Beyond what an osquery package row
+ * gives, an apk record knows its licence and home page, which become
+ * `licenses` and a `website` external reference.
+ *
+ * @param {Object[]} packages Packages from readApkInstalledDb()
+ * @returns {Object[]} Components
+ */
+export function convertApkPackages(packages) {
+  const components = [];
+  for (const pkg of packages || []) {
+    const row = { name: pkg.name, version: pkg.version };
+    for (const column of ["arch", "origin", "maintainer"]) {
+      if (pkg[column]) {
+        row[column] = pkg[column];
+      }
+    }
+    const [component] = convertOSQueryResults(
+      "apk_packages",
+      { purlType: "apk" },
+      [row],
+    );
+    if (!component) {
+      continue;
+    }
+    // The description is set here and not in the row. The GTFOBins matcher reads
+    // a row's description for command names, which a deb or rpm row has no
+    // column for, and the free text of an apk description matches by chance:
+    // "Alpine base dir structure" is no use of the alpine mail client.
+    if (pkg.description) {
+      component.description = pkg.description;
+    }
+    const licenses = osPackageLicenses(pkg.license, /\s+/u);
+    if (licenses.length) {
+      component.licenses = licenses;
+    }
+    const website = osPackageWebsite(pkg.url);
+    if (website) {
+      component.externalReferences = [{ type: "website", url: website }];
+    }
+    components.push(component);
+  }
+  return components;
+}
+
+// The frontmatter and osquery columns of a skill row that become properties,
+// named as the ai-skill project scan names them where it has a name.
+const AGENT_SKILL_PROPERTY_NAMES = [
+  ["name", "cdx:skill:name"],
+  ["description", "cdx:skill:description"],
+  ["license", "cdx:skill:license"],
+  ["compatibility", "cdx:skill:compatibility"],
+  ["agent", "cdx:skill:agent"],
+  ["scope", "cdx:skill:scope"],
+  ["allowed_tools", "cdx:skill:allowedTools"],
+  ["resource_count", "cdx:skill:resourceCount"],
+  ["script_count", "cdx:skill:scriptCount"],
+  ["username", "username"],
+];
+const SHA256_HEX = /^[0-9a-f]{64}$/u;
+
+// A skill's licence is free text ("Complete terms in LICENSE.txt"), so it
+// becomes `licenses` only when it is an SPDX id or expression.
+function agentSkillLicenses(text) {
+  const license = `${text ?? ""}`.trim();
+  if (!license) {
+    return [];
+  }
+  if (isExplicitSpdxExpression(license)) {
+    return [{ expression: license }];
+  }
+  const id = findLicenseId(license);
+  return spdxLicenses.includes(id) ? [{ license: { id } }] : [];
+}
+
+/**
+ * Convert the rows of the `ai_agent_skills` query into components.
+ *
+ * A skill installed on the host is what the `ai-skill` project scan finds in a
+ * repository, so it is a `file` component with the same `cdx:file:kind` and
+ * `cdx:skill:*` properties, and no purl: a skill has no package identity. Its
+ * bom-ref is the SHA-256 of its SKILL.md rather than the path, which is under a
+ * home directory, and where it was found is an occurrence. The same skill
+ * found under several roots, such as a plugin marketplace checkout and the
+ * plugin cache, is one component with one occurrence for each.
+ *
+ * @param {Object[]} results Rows of `ai_agent_skills` joined with `hash`
+ * @returns {Object[]} Components
+ */
+export function convertAgentSkillRows(results) {
+  const components = [];
+  for (const row of results || []) {
+    const skillFile = `${row.path ?? ""}`;
+    const directory = `${row.directory ?? ""}` || dirname(skillFile);
+    // A SKILL.md whose frontmatter osquery cannot read has no name; the
+    // project scan names such a skill after its directory too.
+    const name = sanitizeOsQueryIdentity(row.name || basename(directory));
+    if (!skillFile || !name) {
+      continue;
+    }
+    const version = row.version || "";
+    const sha256 = SHA256_HEX.test(`${row.sha256 ?? ""}`)
+      ? row.sha256
+      : undefined;
+    const properties = [
+      { name: "cdx:osquery:category", value: "ai_agent_skills" },
+      { name: "cdx:file:kind", value: "skill-file" },
+      { name: "cdx:agent:inventorySource", value: "osquery" },
+    ];
+    for (const [column, propertyName] of AGENT_SKILL_PROPERTY_NAMES) {
+      const value = `${row[column] ?? ""}`.trim();
+      if (value && value !== "null") {
+        properties.push({ name: propertyName, value });
+      }
+    }
+    if (Number(row.script_count) > 0) {
+      properties.push({ name: "cdx:skill:hasScripts", value: "true" });
+    }
+    properties.push({
+      name: "cdx:skill:specValid",
+      value: isAgentSkillSpecValid(row) ? "true" : "false",
+    });
+    const component = {
+      name,
+      version,
+      description: row.description || "",
+      type: "file",
+      "bom-ref": createOsQueryFallbackBomRef(
+        "ai_agent_skills",
+        "file",
+        name,
+        version,
+        sha256 ? "sha256" : "scope",
+        sha256 || `${row.agent || ""}/${row.scope || ""}`,
+      ),
+      properties,
+    };
+    if (sha256) {
+      component.hashes = [{ alg: "SHA-256", content: sha256 }];
+    }
+    const licenses = agentSkillLicenses(row.license);
+    if (licenses.length) {
+      component.licenses = licenses;
+    }
+    const occurrence = createOccurrenceEvidence(skillFile);
+    if (occurrence) {
+      component.evidence = { occurrences: [occurrence] };
+    }
+    components.push(component);
+  }
+  return mergeLocatedPackages(components);
 }
 function createOsQueryCryptoProperties(queryCategory, res, version) {
   switch (queryCategory) {

package/lib/inventory/fetchBatch.js

@@ -221,7 +221,7 @@ export function openHostCircuit(
     );
   } else if (breaker === "maven-central") {
     console.warn(
-      `${host} ${what}, so cdxgen will not contact it for the next ${minutes} minute(s) and Maven metadata comes from local caches only. Set MAVEN_CENTRAL_URL to a mirror or repository manager, for example https://maven-central.storage-download.googleapis.com/maven2/, to keep license enrichment working.`,
+      `${host} ${what}, so cdxgen will not contact it for the next ${minutes} minute(s) and Maven metadata comes from local caches only. Set MAVEN_CENTRAL_MIRROR=true to use Google's Maven Central mirror, or name a repository manager in MAVEN_CENTRAL_URL, to keep license enrichment working.`,
     );
   } else {
     console.warn(

package/lib/inventory/formulationParsers.js

@@ -4,7 +4,9 @@ import { basename } from "node:path";
 import process from "node:process";
 
 import { readEnvironmentVariable } from "../core/activity.js";
+import { PROJECT_TYPE_ALIASES } from "../core/env.js";
 import { getAllFiles } from "../core/fs.js";
+import { thoughtLog } from "../core/logger.js";
 import { collectOSCryptoLibs } from "./cbomutils.js";
 import { azurePipelinesParser } from "./ciParsers/azurePipelines.js";
 import { circleCiParser } from "./ciParsers/circleCi.js";
@@ -187,6 +189,16 @@ const ENV_BLOCKLIST = [
   "webhook",
 ];
 
+// The project types a scan asked for, as given on the command line.
+function requestedProjectTypes(options) {
+  return [].concat(options?.projectType || []).flatMap((type) =>
+    `${type}`
+      .split(",")
+      .map((part) => part.trim().toLowerCase())
+      .filter(Boolean),
+  );
+}
+
 /**
  * Build the formulation section for a CycloneDX BOM.
  *
@@ -285,14 +297,31 @@ export function addFormulationSection(filePath, options, context = {}) {
     }
   }
 
-  // Build environment details (Java, .NET, Python, Node, GCC, Rust, Go, Ruby)
-  const infoComponents = collectEnvInfo(projectPath);
+  // The toolchains and OS crypto libraries are probed on the machine cdxgen
+  // runs on. A container image or root filesystem was built elsewhere, so the
+  // scanning host's tools say nothing about it.
+  // Only an explicit container type counts: hasAnyProjectType treats
+  // `universal` as every type, and a universal source scan runs on the host.
+  const describesScanningHost = !requestedProjectTypes(options).some((type) =>
+    PROJECT_TYPE_ALIASES.oci.includes(type),
+  );
+  if (!describesScanningHost) {
+    thoughtLog(
+      "Skipping the toolchain and OS crypto library probes for formulation, since they would describe the scanning host and not the image.",
+    );
+  }
+
+  // Build environment details (Java, .NET, Python, Node, GCC, Rust, Go, Ruby,
+  // RubyGems, Bundler, PHP, Composer)
+  const infoComponents = describesScanningHost
+    ? collectEnvInfo(projectPath, options)
+    : [];
   if (infoComponents?.length) {
     components = components.concat(infoComponents);
   }
 
   // OS crypto libraries (cbom mode)
-  if (options.includeCrypto) {
+  if (options.includeCrypto && describesScanningHost) {
     const cryptoLibs = collectOSCryptoLibs(options, context?.executeOsQuery);
     if (cryptoLibs?.length) {
       components = components.concat(cryptoLibs);