@cdxgen/cdxgen 13.5.0 → 13.6.0

package/lib/ecosystems/cppEvidence.js

@@ -19,6 +19,7 @@ import {
 import { build } from "@cdxgen/cdx-purl";
 
 import { DEBUG_MODE } from "../core/activity.js";
+import { readRegularTextFile } from "../core/boundedFile.js";
 import { getAllFiles, safeExistsSync } from "../core/fs.js";
 import { thoughtLog } from "../core/logger.js";
 import { CPP_STD_MODULES } from "../core/state.js";
@@ -29,6 +30,7 @@ import { locateGenericPackage } from "../inventory/purl.js";
 import {
   createIncludeAttributor,
   parseCIncludeSlices,
+  readVcpkgInstalledIndex,
 } from "./cppIncludeResolver.js";
 import {
   parseCmakeLikeFile,
@@ -218,6 +220,243 @@ export function hasCppSources(src, options = {}) {
   );
 }
 
+const MAX_VCPKG_MANIFEST_BYTES = 4 * 1024 * 1024;
+
+/**
+ * The ports vcpkg installed into the `vcpkg_installed` directories of a
+ * project and its build trees, by lower-case port name.
+ *
+ * @param {string} src project directory
+ * @param {Object} options CLI options, for the build-directory candidates
+ * @returns {Map<string, {version: string, triplet: string}>}
+ */
+export function installedVcpkgPorts(src, options = {}) {
+  const ports = new Map();
+  const installedDirs = [src, ...cmakeBuildDirCandidates(src, options)]
+    .map((dir) => join(dir, "vcpkg_installed"))
+    .filter((dir) => safeExistsSync(dir));
+  if (!installedDirs.length) {
+    return ports;
+  }
+  for (const { port, version, triplet, portVersion } of readVcpkgInstalledIndex(
+    installedDirs,
+  ).values()) {
+    if (!ports.has(port.toLowerCase())) {
+      ports.set(port.toLowerCase(), {
+        version,
+        triplet,
+        ...(portVersion ? { portVersion } : {}),
+      });
+    }
+  }
+  return ports;
+}
+
+/**
+ * The version a vcpkg manifest pins for a port in its `overrides`, from any of
+ * the four version fields vcpkg accepts, and the port version, which is a
+ * packaging revision of the same upstream release.
+ * https://learn.microsoft.com/vcpkg/reference/vcpkg-json#overrides
+ *
+ * @param {Object} override one `overrides` entry
+ * @returns {{version: string, portVersion?: string}|undefined}
+ */
+function vcpkgOverrideVersion(override) {
+  if (!override || typeof override !== "object") {
+    return undefined;
+  }
+  const version = [
+    "version",
+    "version-semver",
+    "version-date",
+    "version-string",
+  ]
+    .map((field) => override[field])
+    .find((value) => typeof value === "string" && value.trim());
+  if (!version) {
+    return undefined;
+  }
+  const portVersion = Number(override["port-version"]);
+  return Number.isInteger(portVersion) && portVersion > 0
+    ? { version: version.trim(), portVersion: String(portVersion) }
+    : { version: version.trim() };
+}
+
+/**
+ * Read a vcpkg manifest (vcpkg.json) in a directory: the project it describes
+ * and the ports it declares. A port's version comes from what vcpkg installed
+ * in the build tree when that is known, otherwise from the manifest's
+ * `overrides`, which pin it exactly. A `version>=` constraint is only a
+ * minimum, so it is kept as a property, never as the version.
+ * https://learn.microsoft.com/vcpkg/reference/vcpkg-json
+ *
+ * The read is cheap and runs no tool, so it does not wait for the atom-based
+ * analysis that getCppModules adds.
+ *
+ * @param {string} src directory that may hold vcpkg.json
+ * @param {Object} [options] options
+ * @param {Map<string, {version: string, triplet: string}>} [options.installedPorts]
+ *   ports vcpkg installed, by lower-case name
+ * @param {function(string): boolean} [options.isListed] whether another source
+ *   already lists a port by that name
+ * @returns {{parentComponent: Object|undefined, pkgList: Object[]}}
+ */
+export function parseVcpkgManifest(src, options = {}) {
+  const pkgType = "generic";
+  const result = { parentComponent: undefined, pkgList: [] };
+  const manifestFile = join(src, "vcpkg.json");
+  if (!safeExistsSync(manifestFile)) {
+    return result;
+  }
+  let vcPkgData;
+  try {
+    // A regular file only: a FIFO named vcpkg.json would block a plain read.
+    vcPkgData = JSON.parse(
+      readRegularTextFile(manifestFile, MAX_VCPKG_MANIFEST_BYTES) ?? "",
+    );
+  } catch {
+    return result;
+  }
+  if (!vcPkgData || typeof vcPkgData !== "object") {
+    return result;
+  }
+  const installedPorts = options.installedPorts || new Map();
+  const isListed = options.isListed || (() => false);
+  const overrides = new Map();
+  if (Array.isArray(vcPkgData.overrides)) {
+    for (const override of vcPkgData.overrides) {
+      const version = vcpkgOverrideVersion(override);
+      if (typeof override?.name === "string" && version) {
+        overrides.set(override.name.toLowerCase(), version);
+      }
+    }
+  }
+  if (vcPkgData.name) {
+    const parentPurl = build({
+      type: pkgType,
+      namespace: null,
+      name: vcPkgData.name,
+      version: vcPkgData.version || vcPkgData["version-string"] || null,
+    });
+    const parentComponent = {
+      name: vcPkgData.name,
+      version: vcPkgData.version || vcPkgData["version-string"] || "",
+      description: vcPkgData.description,
+      license: vcPkgData.license,
+      purl: parentPurl,
+      type: "application",
+      "bom-ref": decodeURIComponent(parentPurl),
+    };
+    if (vcPkgData.homepage) {
+      parentComponent.homepage = { url: vcPkgData.homepage };
+    }
+    // The builtin baseline is the vcpkg commit that pins every port version,
+    // so it identifies the version set the manifest resolves against.
+    if (typeof vcPkgData["builtin-baseline"] === "string") {
+      parentComponent.properties = [
+        {
+          name: "cdx:vcpkg:baseline",
+          value: vcPkgData["builtin-baseline"],
+        },
+      ];
+    }
+    result.parentComponent = parentComponent;
+  }
+  if (!Array.isArray(vcPkgData.dependencies)) {
+    return result;
+  }
+  const added = new Set();
+  for (const avcdep of vcPkgData.dependencies) {
+    let avcpkgName;
+    let scope;
+    let vcpkgFeatures;
+    let minimumVersion;
+    if (typeof avcdep === "string" || avcdep instanceof String) {
+      avcpkgName = `${avcdep}`;
+    } else if (avcdep && typeof avcdep === "object" && avcdep.name) {
+      avcpkgName = avcdep.name;
+      if (avcdep.host) {
+        scope = "optional";
+      }
+      vcpkgFeatures = Array.isArray(avcdep.features)
+        ? avcdep.features
+            .map((feature) =>
+              typeof feature === "string" ? feature : feature?.name,
+            )
+            .filter(Boolean)
+            .join(",")
+        : undefined;
+      if (typeof avcdep["version>="] === "string") {
+        minimumVersion = avcdep["version>="];
+      }
+    }
+    // Is this a dependency we haven't seen before including the all lower and upper case version?
+    if (!avcpkgName || isListed(avcpkgName) || added.has(avcpkgName)) {
+      continue;
+    }
+    added.add(avcpkgName);
+    // the version vcpkg installed, when the build tree has the port, else the
+    // version the manifest's overrides pin
+    const installed = installedPorts.get(avcpkgName.toLowerCase());
+    const overridden = overrides.get(avcpkgName.toLowerCase());
+    const version = installed?.version || overridden?.version || "";
+    const pkgPurl = build({
+      type: pkgType,
+      namespace: null,
+      name: avcpkgName,
+      version: version || null,
+    });
+    result.pkgList.push({
+      group: "",
+      name: avcpkgName,
+      type: pkgType,
+      version,
+      purl: pkgPurl,
+      scope,
+      "bom-ref": decodeURIComponent(pkgPurl),
+      properties: [
+        { name: "cdx:vcpkg:declared", value: "true" },
+        ...(vcpkgFeatures
+          ? [{ name: "cdx:vcpkg:features", value: vcpkgFeatures }]
+          : []),
+        ...(installed
+          ? [{ name: "cdx:vcpkg:triplet", value: installed.triplet }]
+          : []),
+        ...(!installed && overridden
+          ? [{ name: "cdx:vcpkg:versionSource", value: "override" }]
+          : []),
+        ...((installed ? installed.portVersion : overridden?.portVersion)
+          ? [
+              {
+                name: "cdx:vcpkg:portVersion",
+                value: installed
+                  ? installed.portVersion
+                  : overridden.portVersion,
+              },
+            ]
+          : []),
+        ...(minimumVersion
+          ? [{ name: "cdx:vcpkg:minimumVersion", value: minimumVersion }]
+          : []),
+      ],
+      evidence: {
+        identity: {
+          field: "purl",
+          confidence: 0.5,
+          methods: [
+            {
+              technique: "source-code-analysis",
+              confidence: 0.5,
+              value: `Filename ${manifestFile}`,
+            },
+          ],
+        },
+      },
+    });
+  }
+  return result;
+}
+
 /**
  * Method to find c/c++ modules by collecting usages with atom
  *
@@ -263,111 +502,24 @@ export function getCppModules(
   const vcpkgPorts = attributeInclude.vcpkgPorts();
   // Let's look for any vcpkg.json file to tell us about the directory we're scanning
   // users can use this file to give us a clue even if they do not use vcpkg library manager
-  if (safeExistsSync(join(src, "vcpkg.json"))) {
-    const vcPkgData = JSON.parse(
-      readFileSync(join(src, "vcpkg.json"), { encoding: "utf-8" }),
-    );
-    if (vcPkgData && Object.keys(vcPkgData).length && vcPkgData.name) {
-      const parentPurl = build({
-        type: pkgType,
-        namespace: "" || null,
-        name: vcPkgData.name,
-        version: vcPkgData.version || "" || null,
-      });
-      parentComponent = {
-        name: vcPkgData.name,
-        version: vcPkgData.version || "",
-        description: vcPkgData.description,
-        license: vcPkgData.license,
-        purl: parentPurl,
-        type: "application",
-        "bom-ref": decodeURIComponent(parentPurl),
-      };
-      if (vcPkgData.homepage) {
-        parentComponent.homepage = { url: vcPkgData.homepage };
-      }
-      // The builtin baseline is the vcpkg commit that pins every port version,
-      // so it identifies the version set the manifest resolves against.
-      if (typeof vcPkgData["builtin-baseline"] === "string") {
-        parentComponent.properties = [
-          {
-            name: "cdx:vcpkg:baseline",
-            value: vcPkgData["builtin-baseline"],
-          },
-        ];
-      }
-      // Are there any dependencies declared in vcpkg.json
-      if (vcPkgData.dependencies && Array.isArray(vcPkgData.dependencies)) {
-        for (const avcdep of vcPkgData.dependencies) {
-          let avcpkgName;
-          let scope;
-          let vcpkgFeatures;
-          if (typeof avcdep === "string" || avcdep instanceof String) {
-            avcpkgName = avcdep;
-          } else if (Object.keys(avcdep).length && avcdep.name) {
-            avcpkgName = avcdep.name;
-            if (avcdep.host) {
-              scope = "optional";
-            }
-            vcpkgFeatures = Array.isArray(avcdep.features)
-              ? avcdep.features.join(",")
-              : undefined;
-          }
-          // Is this a dependency we haven't seen before including the all lower and upper case version?
-          if (
-            avcpkgName &&
-            !epkgMap[epkgKey("", avcpkgName)] &&
-            !epkgMap[epkgKey("", avcpkgName.toLowerCase())] &&
-            !epkgMap[epkgKey("", avcpkgName.toUpperCase())]
-          ) {
-            // the version vcpkg installed, when the build tree has the port
-            const installed = vcpkgPorts.get(avcpkgName.toLowerCase());
-            const pkgPurl = build({
-              type: pkgType,
-              namespace: "" || null,
-              name: avcpkgName,
-              version: installed?.version || null,
-            });
-            const apkg = {
-              group: "",
-              name: avcpkgName,
-              type: pkgType,
-              version: installed?.version || "",
-              purl: pkgPurl,
-              scope,
-              "bom-ref": decodeURIComponent(pkgPurl),
-              properties: [
-                { name: "cdx:vcpkg:declared", value: "true" },
-                ...(vcpkgFeatures
-                  ? [{ name: "cdx:vcpkg:features", value: vcpkgFeatures }]
-                  : []),
-                ...(installed
-                  ? [{ name: "cdx:vcpkg:triplet", value: installed.triplet }]
-                  : []),
-              ],
-              evidence: {
-                identity: {
-                  field: "purl",
-                  confidence: 0.5,
-                  methods: [
-                    {
-                      technique: "source-code-analysis",
-                      confidence: 0.5,
-                      value: `Filename ${join(src, "vcpkg.json")}`,
-                    },
-                  ],
-                },
-              },
-            };
-            if (!pkgAddedMap[avcpkgName]) {
-              pkgList.push(apkg);
-              dependsOn.add(apkg["bom-ref"]);
-              pkgAddedMap[avcpkgName] = true;
-            }
-          }
-        }
-      }
-    } // if
+  const manifest = parseVcpkgManifest(src, {
+    installedPorts: vcpkgPorts,
+    isListed: (name) =>
+      Boolean(
+        epkgMap[epkgKey("", name)] ||
+          epkgMap[epkgKey("", name.toLowerCase())] ||
+          epkgMap[epkgKey("", name.toUpperCase())],
+      ),
+  });
+  if (manifest.parentComponent) {
+    parentComponent = manifest.parentComponent;
+  }
+  for (const apkg of manifest.pkgList) {
+    if (!pkgAddedMap[apkg.name]) {
+      pkgList.push(apkg);
+      dependsOn.add(apkg["bom-ref"]);
+      pkgAddedMap[apkg.name] = true;
+    }
   }
   // xmake locks its resolved requirements in `xmake-requires.lock`; the
   // entries are additions to whatever other evidence collectors found.
@@ -464,7 +616,12 @@ export function getCppModules(
           name: target.port,
           version: target.version || null,
         }),
-        properties: [{ name: "cdx:vcpkg:triplet", value: target.triplet }],
+        properties: [
+          { name: "cdx:vcpkg:triplet", value: target.triplet },
+          ...(target.portVersion
+            ? [{ name: "cdx:vcpkg:portVersion", value: target.portVersion }]
+            : []),
+        ],
         technique: "manifest-analysis",
       };
     } else {

package/lib/ecosystems/cppIncludeResolver.js

@@ -70,7 +70,7 @@ export function parseCIncludeSlices(sliceData) {
  * line relative to the installed directory.
  *
  * @param {string[]} installedDirs `vcpkg_installed` directories
- * @returns {Map<string, {port: string, version: string, triplet: string}>}
+ * @returns {Map<string, {port: string, version: string, triplet: string, portVersion?: string}>}
  */
 export function readVcpkgInstalledIndex(installedDirs) {
   const index = new Map();
@@ -90,7 +90,9 @@ export function readVcpkgInstalledIndex(installedDirs) {
         continue;
       }
       const port = stem.slice(0, first);
-      const version = stem.slice(first + 1, last);
+      // vcpkg spells a port version as `<version>#<port-version>`; the port
+      // version is a packaging revision of the same upstream release.
+      const [version, portVersion] = stem.slice(first + 1, last).split("#");
       const triplet = stem.slice(last + 1);
       let text = "";
       try {
@@ -103,7 +105,12 @@ export function readVcpkgInstalledIndex(installedDirs) {
         if (!rel || rel.endsWith("/")) {
           continue;
         }
-        index.set(resolve(dir, rel), { port, version, triplet });
+        index.set(resolve(dir, rel), {
+          port,
+          version,
+          triplet,
+          ...(portVersion ? { portVersion } : {}),
+        });
       }
     }
   }
@@ -282,9 +289,13 @@ export function createIncludeAttributor(context) {
   attribute.vcpkgPorts = () => {
     const ports = new Map();
     if (installedDirs.length) {
-      for (const { port, version, triplet } of vcpkg().values()) {
+      for (const { port, version, triplet, portVersion } of vcpkg().values()) {
         if (!ports.has(port)) {
-          ports.set(port, { version, triplet });
+          ports.set(port, {
+            version,
+            triplet,
+            ...(portVersion ? { portVersion } : {}),
+          });
         }
       }
     }

package/lib/ecosystems/ecosystems.js

@@ -77,6 +77,7 @@ import {
   findMavenCoordinatesBySha1,
   inferMavenCoordinatesFromPath,
 } from "../inventory/jvmLocalRepos.js";
+import { mavenCentralFetchUrl } from "../inventory/mavenCentralMirror.js";
 import {
   applyPurl,
   genericPurl,
@@ -92,6 +93,7 @@ import {
   guessLicenseId,
   spdxLicenses,
 } from "../inventory/spdx.js";
+import { parseSpdxExpression } from "../inventory/spdxExpression.js";
 import { extractLicenseText, extractRepoUrl } from "../parsers/htmlExtract.js";
 import { xml2js } from "../parsers/xml.js";
 import {
@@ -1050,7 +1052,8 @@ export async function getMvnMetadata(
   }
   const repos = {
     MAVEN_CENTRAL_URL: withTrailingSlash(
-      readEnvironmentVariable("MAVEN_CENTRAL_URL") ||
+      mavenCentralFetchUrl() ||
+        readEnvironmentVariable("MAVEN_CENTRAL_URL") ||
         "https://repo1.maven.org/maven2/",
     ),
     ANDROID_MAVEN_URL: withTrailingSlash(
@@ -1771,6 +1774,10 @@ export function parseBdistMetadata(mDataFile, rawMetadata = undefined) {
   }
   const lines = mData.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split("\n");
   let isBody = false;
+  // PEP 639 (metadata 2.4) `License-Expression` is a validated SPDX expression
+  // that wins over the free-text `License` field whichever header comes first.
+  let licenseExpression;
+  let licenseName;
   for (const line of lines) {
     if (line.trim() === "") {
       isBody = true;
@@ -1797,17 +1804,11 @@ export function parseBdistMetadata(mDataFile, rawMetadata = undefined) {
         pkg.author = value;
         break;
       case "license-expression":
-        pkg.licenses.push({
-          expression: value,
-        });
+        licenseExpression = value;
         break;
       case "license":
-        if (value !== "UNKNOWN" && pkg.licenses.length === 0) {
-          pkg.licenses.push({
-            license: {
-              name: value,
-            },
-          });
+        if (value !== "UNKNOWN" && licenseName === undefined) {
+          licenseName = value;
         }
         break;
       case "home-page":
@@ -1867,6 +1868,20 @@ export function parseBdistMetadata(mDataFile, rawMetadata = undefined) {
         break;
     }
   }
+  if (licenseExpression && parseSpdxExpression(licenseExpression).valid) {
+    // PEP 639 guarantees a valid SPDX expression here, including `LicenseRef-`
+    // document references, which travel as an expression and never as an id.
+    pkg.licenses = [{ expression: licenseExpression }];
+  } else {
+    if (licenseExpression && DEBUG_MODE) {
+      console.log(
+        `License-Expression "${licenseExpression}" of ${pkg.name} is not a valid SPDX expression, so the older license fields are used`,
+      );
+    }
+    if (licenseName) {
+      pkg.licenses = [{ license: { name: licenseName } }];
+    }
+  }
   if (mDataFile) {
     pkg.evidence = {
       identity: {