@cdxgen/cdxgen 13.2.0 → 13.4.0

package/lib/ecosystems/parsers-go.js

@@ -1,15 +1,20 @@
 import { Buffer } from "node:buffer";
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
+import { readdirSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { delimiter, dirname, join } from "node:path";
 
 import { Purl } from "@cdxgen/cdx-purl";
 
-import { DEBUG_MODE } from "../core/activity.js";
+import { DEBUG_MODE, readEnvironmentVariable } from "../core/activity.js";
 import { shouldFetchLicense, shouldFetchVCS } from "../core/env.js";
-import { safeExistsSync } from "../core/fs.js";
-import { applyPurl, tryBuildPurl, tryParsePurl } from "../inventory/purl.js";
 import {
-  _clearMetadataCache,
+  applyPurl,
+  canonicalPurlFromLooseString,
+  tryBuildPurl,
+  tryParsePurl,
+} from "../inventory/purl.js";
+import { guessLicenseId } from "../inventory/spdx.js";
+import {
   getGoPkgLicense,
   getGoPkgVCSUrl,
   prefetchGoPkgMetadata,
@@ -25,6 +30,163 @@ function toBase64(hexString) {
   return Buffer.from(hexString, "hex").toString("base64");
 }
 
+/**
+ * The root of the Go module cache on this machine.
+ *
+ * @returns {string|undefined} `$GOMODCACHE`, or `$GOPATH/pkg/mod`, or the
+ *   default `~/go/pkg/mod`.
+ */
+function goModuleCacheDir() {
+  const gomodcache = readEnvironmentVariable("GOMODCACHE");
+  if (gomodcache) {
+    return gomodcache;
+  }
+  const gopath = readEnvironmentVariable("GOPATH");
+  if (gopath) {
+    return join(gopath.split(delimiter)[0], "pkg", "mod");
+  }
+  return join(homedir(), "go", "pkg", "mod");
+}
+
+/**
+ * Escape a module path the way Go's module cache directory names do: every
+ * upper-case letter becomes `!` followed by the lower-case one, so the cache
+ * stays case-preserving on case-insensitive file systems
+ * (`github.com/Azure` is stored as `github.com/!azure`).
+ *
+ * @param {string} modulePath Full module path
+ * @returns {string} The escaped path
+ */
+function escapeGoModulePath(modulePath) {
+  let escaped = "";
+  for (const ch of modulePath) {
+    if (ch >= "A" && ch <= "Z") {
+      escaped += `!${ch.toLowerCase()}`;
+    } else {
+      escaped += ch;
+    }
+  }
+  return escaped;
+}
+
+/**
+ * Whether a file name is a licence or copying notice, whose text describes the
+ * module's licence without a registry round.
+ *
+ * @param {string} fileName Candidate file name
+ * @returns {boolean}
+ */
+function isGoLicenseFileName(fileName) {
+  const lower = fileName.toLowerCase();
+  return lower.startsWith("license") || lower.startsWith("copying");
+}
+
+/**
+ * The licence or copying notice of a module that is already on this machine,
+ * from the directory `go list` reported, from the project's `vendor/` tree, or
+ * from the module cache. Reading it costs no network round trip, so it answers
+ * before pkg.go.dev is asked.
+ *
+ * The directory `go list` reported is always read. The vendor tree and the
+ * module cache stand in for pkg.go.dev, so they are read only when licences
+ * are being fetched.
+ *
+ * @param {string} name Full module path
+ * @param {string} [version] Module version, for the module cache layout
+ * @param {Object} [options] Options
+ * @param {string} [options.dir] The module's directory, as `go list` reports it
+ * @param {string} [options.projectRoot] Project root holding a `vendor/` tree
+ * @returns {string|undefined} The notice text, or undefined when none is held
+ *   locally
+ */
+export function readLocalGoLicense(name, version, options = {}) {
+  if (!name) {
+    return undefined;
+  }
+  const candidateDirs = [];
+  if (options.dir) {
+    candidateDirs.push(options.dir);
+  }
+  const fetchingLicenses = shouldFetchLicense();
+  if (fetchingLicenses && options.projectRoot) {
+    candidateDirs.push(join(options.projectRoot, "vendor", name));
+  }
+  if (fetchingLicenses && version) {
+    const cacheRoot = goModuleCacheDir();
+    if (cacheRoot) {
+      candidateDirs.push(
+        join(cacheRoot, `${escapeGoModulePath(name)}@${version}`),
+      );
+    }
+  }
+  for (const dir of candidateDirs) {
+    let entries;
+    try {
+      entries = readdirSync(dir);
+    } catch {
+      continue;
+    }
+    const names = entries
+      .filter((e) => isGoLicenseFileName(e))
+      .sort((a, b) => a.localeCompare(b));
+    // A directory can hold both a licence and a copying notice; the licence
+    // notice wins when both are present.
+    const chosen =
+      names.find((f) => f.toLowerCase().startsWith("license")) ||
+      names.find((f) => f.toLowerCase().startsWith("copying"));
+    if (!chosen) {
+      continue;
+    }
+    try {
+      const text = readFileSync(join(dir, chosen), {
+        encoding: "utf-8",
+      });
+      if (text?.length) {
+        return text;
+      }
+    } catch {
+      // An unreadable notice is the same as none: the registry path answers.
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Mark the modules whose licence notice is held on this machine, so the
+ * pkg.go.dev batch skips their licence tab but still fetches their module page
+ * for the repository URL.
+ *
+ * @param {Object[]} modules Modules with `name`, `version` and the options
+ *   {@link readLocalGoLicense} takes
+ * @returns {Object[]} Copies of the modules with `licenseKnown` set
+ */
+function withLocalLicenseFlags(modules) {
+  return modules.map((m) => ({
+    ...m,
+    licenseKnown: !!readLocalGoLicense(m.name, m.version, m),
+  }));
+}
+
+/**
+ * The licence object for a notice read from disk: its SPDX id when the text
+ * names one, else a CUSTOM licence, with the notice as its text either way.
+ *
+ * @param {string} text Notice text
+ * @returns {Object} CycloneDX licence object
+ */
+function localGoLicenseObject(text) {
+  const licenseId = guessLicenseId(text);
+  const identity = !licenseId
+    ? { name: "CUSTOM" }
+    : licenseId.includes(" ")
+      ? { name: licenseId }
+      : { id: licenseId };
+  return {
+    ...identity,
+    text: { contentType: "text/plain", content: text },
+  };
+}
+
 /**
  * Builds a Go package component object containing purl, bom-ref, integrity hash,
  * and optionally license and VCS external reference information.
@@ -33,11 +195,31 @@ function toBase64(hexString) {
  * @param {string} name Package name (full module path when group is empty)
  * @param {string} version Package version string
  * @param {string} hash Integrity hash (e.g. "sha256-…"), used as _integrity
+ * @param {Object} [options] Options
+ * @param {boolean} [options.skipRegistryLookups] Leave the licence and VCS
+ *   URL unset instead of asking pkg.go.dev. Used for the main module, which
+ *   no registry holds.
+ * @param {string} [options.dir] The module's directory, as `go list` reports
+ *   it, where a local licence notice is read first
+ * @param {string} [options.projectRoot] Project root holding a `vendor/` tree
+ *   the module may be vendored in
  * @returns {Promise<Object>} Component object ready for inclusion in a BOM package list
  */
-export async function getGoPkgComponent(group, name, version, hash) {
+export async function getGoPkgComponent(
+  group,
+  name,
+  version,
+  hash,
+  options = {},
+) {
   let license;
-  if (shouldFetchLicense()) {
+  let licenses;
+  // A notice held on this machine answers before pkg.go.dev is asked, and the
+  // registry is consulted only when there is none.
+  const localLicense = readLocalGoLicense(name, version, options);
+  if (localLicense) {
+    licenses = [{ license: localGoLicenseObject(localLicense) }];
+  } else if (shouldFetchLicense() && !options.skipRegistryLookups) {
     if (DEBUG_MODE) {
       console.log(
         `About to fetch go package license information for ${group}:${name}`,
@@ -46,6 +228,7 @@ export async function getGoPkgComponent(group, name, version, hash) {
     license = await getGoPkgLicense({
       group: group,
       name: name,
+      version: version,
     });
   }
   // Split the full module path into namespace and name, since the caller
@@ -71,8 +254,8 @@ export async function getGoPkgComponent(group, name, version, hash) {
       version: version || null,
     })?.replace(/%2F/g, "/") || null;
   let vcs;
-  if (shouldFetchVCS()) {
-    vcs = await getGoPkgVCSUrl(group, name);
+  if (shouldFetchVCS() && !options.skipRegistryLookups) {
+    vcs = await getGoPkgVCSUrl(group, name, version);
   }
   const packageInfo = {
     group: group,
@@ -81,6 +264,9 @@ export async function getGoPkgComponent(group, name, version, hash) {
     _integrity: hash,
     license: license,
   };
+  if (licenses) {
+    packageInfo.licenses = licenses;
+  }
   applyPurl(packageInfo, purlString);
   if (vcs) {
     packageInfo.externalReferences = [{ type: "vcs", url: vcs }];
@@ -104,7 +290,9 @@ export async function parseGoModData(goModData, gosumMap) {
     return {};
   }
   const { parentComponent, entries } = decodeGoModData(goModData, gosumMap);
-  await prefetchGoPkgMetadata(entries);
+  // A module whose licence notice is read locally is not asked for its
+  // licence; getGoPkgComponent reads the same notice for each entry below.
+  await prefetchGoPkgMetadata(withLocalLicenseFlags(entries));
   for (const entry of entries) {
     const component = await getGoPkgComponent(
       "",
@@ -124,8 +312,6 @@ export async function parseGoModData(goModData, gosumMap) {
     }
     pkgComponentsList.push(component);
   }
-  // Clear the cache
-  _clearMetadataCache();
   return {
     parentComponent,
     pkgList: pkgComponentsList.sort((a, b) =>
@@ -266,18 +452,30 @@ export async function parseGoModulesTxt(txtFile, gosumMap) {
   const pkgs = txtData
     .split("\n")
     .filter((p) => p.trim().replace(/["']/g, "").startsWith("# "));
-  await prefetchGoPkgMetadata(
-    // # cel.dev/expr v0.18.0
-    pkgs.map((l) => ({ group: "", name: l.split(" ")[1] })),
-  );
-  for (const l of pkgs) {
+  // A modules.txt file lives in the vendor/ tree, so the modules it lists are
+  // vendored one directory up from it.
+  const projectRoot = dirname(dirname(txtFile));
+  const modules = pkgs.map((l) => {
     const tmpA = l.split(" ");
-    if (!tmpA.length === 3) {
-      continue;
-    }
-    const version = tmpA[2];
-    const gosumHash = gosumMap[`${tmpA[1]}@${version}`];
-    const component = await getGoPkgComponent("", tmpA[1], tmpA[2], gosumHash);
+    return {
+      group: "",
+      name: tmpA[1],
+      version: tmpA[2],
+      projectRoot,
+    };
+  });
+  // A module whose vendored licence notice is read is not asked for its
+  // licence; getGoPkgComponent reads the same file.
+  await prefetchGoPkgMetadata(withLocalLicenseFlags(modules));
+  for (const m of modules) {
+    const gosumHash = gosumMap[`${m.name}@${m.version}`];
+    const component = await getGoPkgComponent(
+      "",
+      m.name,
+      m.version,
+      gosumHash,
+      { projectRoot },
+    );
     let confidence = 0.7;
     if (gosumHash) {
       component.scope = "required";
@@ -308,19 +506,44 @@ export async function parseGoListDep(rawOutput, gosumMap) {
     // covers every module the loop resolves and nothing else.
     const prefetchSeen = new Set();
     const prefetchModules = [];
+    // The directory of the main module row, which is the project root, under
+    // which the vendored copies of the dependencies live. `go list` may print
+    // the main module last, so its row is found before the batch is built.
+    let mainModuleDir;
+    for (const l of pkgs) {
+      const verArr = l.trim().replace(/["']/g, "").split("|");
+      if (verArr?.length >= 10 && verArr[5] === "true" && verArr[9]?.length) {
+        mainModuleDir = verArr[9];
+      }
+    }
     for (const l of pkgs) {
       const verArr = l.trim().replace(/["']/g, "").split("|");
       if (!verArr || verArr.length < 5) {
         continue;
       }
+      // The row whose fifth field is true is the main module: the project
+      // itself, which no registry holds and which the loop below turns into
+      // the parent component.
+      if (verArr[5] === "true") {
+        continue;
+      }
       const key = `${verArr[0]}-${verArr[1]}`;
       if (prefetchSeen.has(key)) {
         continue;
       }
       prefetchSeen.add(key);
-      prefetchModules.push({ group: "", name: verArr[0] });
+      prefetchModules.push({
+        group: "",
+        name: verArr[0],
+        version: verArr[1],
+        dir: verArr[9]?.length ? verArr[9] : undefined,
+        projectRoot: mainModuleDir,
+      });
     }
-    await prefetchGoPkgMetadata(prefetchModules);
+    // A module whose licence notice is held on this machine is not asked for
+    // its licence: getGoPkgComponent reads the same notices, so the batch
+    // covers exactly what the loop will look up.
+    await prefetchGoPkgMetadata(withLocalLicenseFlags(prefetchModules));
     for (const l of pkgs) {
       const verArr = l.trim().replace(/["']/g, "").split("|");
       if (verArr && verArr.length >= 5) {
@@ -333,11 +556,17 @@ export async function parseGoListDep(rawOutput, gosumMap) {
           if (!gosumHash && verArr.length >= 8 && verArr[8]?.length) {
             gosumHash = `sha256-${verArr[8].replace("h1:", "")}`;
           }
+          const isMainModule = verArr[5] === "true";
           const component = await getGoPkgComponent(
             "",
             verArr[0],
             version,
             gosumHash,
+            {
+              skipRegistryLookups: isMainModule,
+              dir: verArr[9]?.length ? verArr[9] : undefined,
+              projectRoot: mainModuleDir,
+            },
           );
           // This is misusing the scope attribute to represent direct vs indirect
           if (verArr[2] === "false") {
@@ -380,21 +609,6 @@ export async function parseGoListDep(rawOutput, gosumMap) {
               name: "cdx:go:local_dir",
               value: verArr[9],
             });
-            if (safeExistsSync(join(verArr[9], "LICENSE"))) {
-              const licenseText = readFileSync(join(verArr[9], "LICENSE"), {
-                encoding: "utf-8",
-              });
-              if (licenseText?.length) {
-                component.licenses = [
-                  {
-                    license: {
-                      name: "CUSTOM",
-                      text: { contentType: "text/plain", content: licenseText },
-                    },
-                  },
-                ];
-              }
-            }
           }
           if (verArr.length > 5 && verArr[5] === "true") {
             parentComponent = component;
@@ -443,6 +657,23 @@ function _addGoComponentEvidence(component, goModFile, confidence = 0.8) {
   return component;
 }
 
+/**
+ * Build the purl of one `module@version` token from `go mod graph` output.
+ *
+ * The version is written the way Go prints it, so `+incompatible` and the build
+ * metadata of a pseudo-version are unescaped. Reading the token strictly would
+ * reject it and leave a reference that keeps the module path's upper-case
+ * letters, which then never matches the lower-cased component and drops the
+ * dependency edge.
+ *
+ * @param {string} token Module path, optionally followed by `@version`
+ * @returns {Purl|null} Parsed purl, or null when the token forms no valid golang purl
+ */
+function goModGraphTokenPurl(token) {
+  const canonical = canonicalPurlFromLooseString(`pkg:golang/${token}`);
+  return canonical ? Purl.parse(canonical) : null;
+}
+
 /**
  * Parse go mod graph
  *
@@ -512,22 +743,20 @@ export async function parseGoModGraph(
           // cdx-purl requires a namespace. Use the raw ref when parsing fails.
           let sourceRefString;
           let sourceName = tmpA[0].split("@")[0];
-          let sourcePurl = null;
-          try {
-            sourcePurl = Purl.parse(`pkg:golang/${tmpA[0]}`);
+          const sourcePurl = goModGraphTokenPurl(tmpA[0]);
+          if (sourcePurl) {
             sourceRefString = decodeURIComponent(sourcePurl.toString());
             sourceName = `${sourcePurl.namespace ? `${sourcePurl.namespace}/` : ""}${sourcePurl.name}`;
-          } catch {
+          } else {
             sourceRefString = `pkg:golang/${tmpA[0]}`;
           }
           let dependsRefString;
           let dependsName = tmpA[1].split("@")[0];
-          let dependsPurl = null;
-          try {
-            dependsPurl = Purl.parse(`pkg:golang/${tmpA[1]}`);
+          const dependsPurl = goModGraphTokenPurl(tmpA[1]);
+          if (dependsPurl) {
             dependsRefString = decodeURIComponent(dependsPurl.toString());
             dependsName = `${dependsPurl.namespace ? `${dependsPurl.namespace}/` : ""}${dependsPurl.name}`;
-          } catch {
+          } else {
             dependsRefString = `pkg:golang/${tmpA[1]}`;
           }
           // Since go mod graph over-reports direct dependencies we use the existing list
@@ -542,11 +771,20 @@ export async function parseGoModGraph(
           }
           // Add the source and depends to the pkgList
           if (!addedPkgs[tmpA[0]] && !excludedRefs.includes(sourceRefString)) {
+            // go mod graph prints a main module without a version. A main
+            // module is the project itself, or a go.work member, which no
+            // registry holds, so it is never looked up. It still becomes a
+            // component, because the graph's edges hang off it.
+            const isParentSource =
+              !tmpA[0].includes("@") ||
+              sourceName === goModPkgMap?.parentComponent?.name ||
+              sourceRefString === parentComponent?.["bom-ref"];
             const component = await getGoPkgComponent(
               "",
               sourceName,
               sourcePurl?.version || tmpA[0].split("@")[1],
               gosumMap[tmpA[0]],
+              { skipRegistryLookups: isParentSource },
             );
             let confidence = 0.7;
             if (goModOptionalDepsMap[component["bom-ref"]]) {
@@ -684,19 +922,49 @@ export function parseGoModWhy(rawOutput) {
 }
 
 /**
- * Parse go sum data
+ * Reports whether `go mod why -m` output says the main module does not need
+ * the module. go prints `(main module does not need module X)`, or
+ * `(main module does not need to vendor module X)` with `-vendor`, and exits
+ * with status 0 either way.
+ *
+ * @param {string} rawOutput Output from go mod why
+ * @returns {boolean} True when go reports the module as not needed
+ */
+export function isGoModWhyNotNeeded(rawOutput) {
+  if (typeof rawOutput !== "string") {
+    return false;
+  }
+  return rawOutput
+    .split("\n")
+    .some((line) => line.trim().startsWith("(main module does not need "));
+}
+
+/**
+ * Read the `module@version` to hash map out of go.sum contents, without
+ * building components or touching the network.
+ *
  * @param {string} gosumData Content of go.sum
- * @returns package list
+ * @returns {Object} Map of `module@version` keys to `sha256-…` values
  */
-export async function parseGosumData(gosumData) {
-  const pkgList = [];
-  if (!gosumData) {
-    return pkgList;
+export function parseGosumHashes(gosumData) {
+  const gosumMap = {};
+  for (const entry of decodeGosumData(gosumData)) {
+    gosumMap[`${entry.name}@${entry.version}`] = entry.hash;
   }
-  // Decoding every line first lets the whole module set be fetched in one
-  // batched round before any component is built. One decode function feeds both,
-  // so the batch cannot ask for a module the loop does not resolve.
+  return gosumMap;
+}
+
+/**
+ * Decode the go.mod lines of a go.sum file into module entries.
+ *
+ * @param {string} gosumData Content of go.sum
+ * @returns {Object[]} Entries with `name`, `version` and `hash`
+ */
+function decodeGosumData(gosumData) {
   const entries = [];
+  if (!gosumData) {
+    return entries;
+  }
   for (const l of gosumData.split("\n")) {
     const m = l.replaceAll("\r", "");
     // look for lines containing go.mod
@@ -710,7 +978,20 @@ export async function parseGosumData(gosumData) {
       });
     }
   }
-  await prefetchGoPkgMetadata(entries);
+  return entries;
+}
+
+/**
+ * Parse go sum data
+ * @param {string} gosumData Content of go.sum
+ * @returns package list
+ */
+export async function parseGosumData(gosumData) {
+  const pkgList = [];
+  const entries = decodeGosumData(gosumData);
+  // A module whose licence notice is read locally is not asked for its
+  // licence; getGoPkgComponent reads the same notice for each entry below.
+  await prefetchGoPkgMetadata(withLocalLicenseFlags(entries));
   for (const entry of entries) {
     pkgList.push(
       await getGoPkgComponent("", entry.name, entry.version, entry.hash),
@@ -737,13 +1018,17 @@ export async function parseGopkgData(gopkgData) {
   for (const l of pkgs) {
     let key = null;
     let value = null;
-    if (l.indexOf("[[projects]]") > -1) {
+    // Every table header ends the project read so far, and only a
+    // [[projects]] table starts another. The keys of [solve-meta] belong to
+    // no project.
+    if (l.trim().startsWith("[")) {
       if (pkg) {
-        pkgList.push(pkg);
+        pkgList.push(await finalizeGopkgProject(pkg));
       }
-      pkg = {};
+      pkg = l.indexOf("[[projects]]") > -1 ? {} : null;
+      continue;
     }
-    if (l.indexOf("=") > -1) {
+    if (pkg && l.indexOf("=") > -1) {
       const tmpA = l.split("=");
       key = tmpA[0].trim();
       value = tmpA[1].trim().replace(/"/g, "");
@@ -756,12 +1041,6 @@ export async function parseGopkgData(gopkgData) {
         case "name":
           pkg.group = "";
           pkg.name = value;
-          if (shouldFetchLicense()) {
-            pkg.license = await getGoPkgLicense({
-              group: pkg.group,
-              name: pkg.name,
-            });
-          }
           break;
         case "version":
           pkg.version = value;
@@ -773,9 +1052,31 @@ export async function parseGopkgData(gopkgData) {
       }
     }
   }
+  // The last project has no header after it.
+  if (pkg) {
+    pkgList.push(await finalizeGopkgProject(pkg));
+  }
   return pkgList;
 }
 
+/**
+ * Attach the licence of one Gopkg project once its name and version are both
+ * known, so the pkg.go.dev URL names the pinned release.
+ *
+ * @param {Object} pkg The project entry read so far
+ * @returns {Promise<Object>} The same entry, with a licence when one was found
+ */
+async function finalizeGopkgProject(pkg) {
+  if (shouldFetchLicense() && pkg.name) {
+    pkg.license = await getGoPkgLicense({
+      group: pkg.group,
+      name: pkg.name,
+      version: pkg.version,
+    });
+  }
+  return pkg;
+}
+
 /**
  * Parses the output of `go version -m` (build info) and returns a list of Go
  * package components for each "dep" line, including name, version, and integrity hash.
@@ -808,7 +1109,7 @@ export async function parseGoVersionData(buildInfoData) {
           : "",
     });
   }
-  await prefetchGoPkgMetadata(entries);
+  await prefetchGoPkgMetadata(withLocalLicenseFlags(entries));
   for (const entry of entries) {
     pkgList.push(
       await getGoPkgComponent("", entry.name, entry.version, entry.hash),