@cdxgen/cdxgen 13.3.0 → 13.5.0

package/lib/inventory/source.js

@@ -1,13 +1,12 @@
 import fs from "node:fs";
 import path from "node:path";
 import process from "node:process";
-import { URL } from "node:url";
+import { fileURLToPath, URL } from "node:url";
 
 import { Purl } from "@cdxgen/cdx-purl";
 import { coerce, diff, prerelease } from "semver";
 
 import {
-  cdxgenAgent,
   createDryRunError,
   DEBUG_MODE,
   hasDangerousUnicode,
@@ -24,11 +23,12 @@ import {
 } from "../core/fs.js";
 import { thoughtLog } from "../core/logger.js";
 import { isValidDriveRoot, isWin } from "../core/paths.js";
+import { registerRunReset } from "../core/runState.js";
 import {
+  gatedGet,
   prefetchEnabled,
   prefetchedResponse,
   prefetchJson,
-  withHostRateLimit,
 } from "./fetchBatch.js";
 import { canonicalPurlFromLooseString } from "./purl.js";
 
@@ -111,6 +111,9 @@ function inferGitOperation(args = []) {
  * @param {string[]} args git arguments
  * @param {Object} options command options
  * @param {string|undefined} options.cwd working directory
+ * @param {boolean|undefined} options.noRepository run outside any repository,
+ *   so the config of a repository in the working directory, such as url.*.insteadOf
+ *   or core.sshCommand, cannot change where a remote command connects
  * @returns {Object} spawn result
  */
 export function hardenedGitCommand(args, options = {}) {
@@ -122,8 +125,14 @@ export function hardenedGitCommand(args, options = {}) {
     GIT_CONFIG_VALUE_0: "false",
     GIT_CONFIG_KEY_1: "safe.bareRepository",
     GIT_CONFIG_VALUE_1: "explicit",
+    GIT_NO_LAZY_FETCH: "1",
     GIT_TERMINAL_PROMPT: "0",
   };
+  if (options.noRepository) {
+    // ls-remote works without a repository. Pointing GIT_DIR at a path that
+    // does not exist stops git from discovering one in the working directory.
+    envConfigs.GIT_DIR = path.join(getTmpDir(), "cdxgen-no-repository.invalid");
+  }
   const env = isSecureMode
     ? {
         ...process.env,
@@ -275,10 +284,14 @@ export function buildReleaseNotesFromGit(repoPath, options = {}) {
     typeof remoteUrl === "string"
       ? validateAndRejectGitSource(remoteUrl)
       : null;
+  // remote.origin.url comes from the scanned repository. Only a remote that
+  // validateAndRejectGitSource recognises and checks may reach ls-remote; git
+  // also treats forms such as host:path as remotes, which skip that check.
   const canDiscoverRemoteTags =
     (!currentTag || !previousTag) &&
     typeof remoteUrl === "string" &&
     !remoteUrl.startsWith("-") &&
+    maybeRemotePath(remoteUrl) &&
     !remoteUrlValidationError &&
     /github\.com[:/]/i.test(remoteUrl);
   if (canDiscoverRemoteTags) {
@@ -298,7 +311,7 @@ export function buildReleaseNotesFromGit(repoPath, options = {}) {
         "--",
         remoteUrl,
       ],
-      {},
+      { noRepository: true },
     );
     const remoteTags = parseLsRemoteTags(remoteTagsResult.stdout);
     if (!currentTag && remoteTags.length) {
@@ -508,6 +521,59 @@ export function isAllowedPath(p) {
   });
 }
 
+/**
+ * Resolve symlinks in a path. When the path does not exist yet, the nearest
+ * existing parent is resolved and the remaining segments are appended, so
+ * /tmp/new on macOS still becomes /private/tmp/new.
+ *
+ * @param {string} p Path string
+ * @returns {string} real path
+ */
+function realPathOrResolved(p) {
+  let current = path.resolve(p);
+  const missing = [];
+  while (true) {
+    try {
+      return path.join(fs.realpathSync(current), ...missing);
+    } catch (_err) {
+      const parent = path.dirname(current);
+      if (parent === current) {
+        return path.resolve(p);
+      }
+      missing.unshift(path.basename(current));
+      current = parent;
+    }
+  }
+}
+
+/**
+ * Checks where the given path really points, after following symlinks, against
+ * where the allowed paths really point. Both sides are resolved, so an allowed
+ * directory that is itself reached through a link (such as /tmp on macOS) still
+ * matches, while a link inside an allowed directory that leads outside it does
+ * not.
+ *
+ * @param {string} p Path string to check
+ * @returns {boolean} true if the real path is inside a real allowed path
+ */
+export function isAllowedRealPath(p) {
+  if (typeof p !== "string" || hasDangerousUnicode(p)) {
+    return false;
+  }
+  const allowedPaths = getAllowedPaths();
+  if (!allowedPaths.length) {
+    return true;
+  }
+  const realP = realPathOrResolved(p);
+  return allowedPaths.some((ap) => {
+    const relativePath = path.relative(realPathOrResolved(ap), realP);
+    return (
+      relativePath === "" ||
+      (!relativePath.startsWith("..") && !path.isAbsolute(relativePath))
+    );
+  });
+}
+
 /**
  * Determine if the path could be a package URL.
  *
@@ -586,6 +652,49 @@ export function validateAndRejectGitSource(filePath) {
       };
     }
 
+    if (hasAmbiguousRemoteCharacters(filePath)) {
+      return {
+        status: 400,
+        error: "Invalid URL Format",
+        details:
+          "The provided Git URL contains backslashes, whitespace or control characters.",
+      };
+    }
+
+    // git reads file:// URLs from the local disk, so they get the path
+    // allowlist rather than the host allowlist.
+    if (gitUrlObj.protocol === "file:") {
+      let localPath;
+      try {
+        localPath = fileURLToPath(gitUrlObj);
+      } catch (_err) {
+        localPath = undefined;
+      }
+      if (
+        gitUrlObj.hostname ||
+        !localPath ||
+        !isAllowedPath(localPath) ||
+        (isWin && !isAllowedWinPath(localPath))
+      ) {
+        return {
+          status: 403,
+          error: "Path Not Allowed",
+          details: "Path is not allowed as per the allowlist.",
+        };
+      }
+      return null;
+    }
+
+    // git, libcurl and OpenSSH must connect to the host that was checked.
+    const gitRemoteHost = getGitRemoteHost(filePath);
+    if (!gitRemoteHost || gitUrlObj.hostname.toLowerCase() !== gitRemoteHost) {
+      return {
+        status: 400,
+        error: "Invalid URL Format",
+        details: "The host of the provided Git URL is ambiguous.",
+      };
+    }
+
     if (!isAllowedHost(gitUrlObj.hostname)) {
       return {
         status: 403,
@@ -598,6 +707,94 @@ export function validateAndRejectGitSource(filePath) {
   return null;
 }
 
+/**
+ * Detect characters that git, libcurl or OpenSSH read differently from the
+ * WHATWG URL parser. A backslash ends the host for new URL() but is part of
+ * the user name for git, and new URL() silently drops tabs and newlines.
+ *
+ * @param {string} remote Git URL or SCP-like address
+ * @returns {boolean} true if the remote contains such characters
+ */
+function hasAmbiguousRemoteCharacters(remote) {
+  if (hasDangerousUnicode(remote)) {
+    return true;
+  }
+  for (const ch of remote) {
+    const code = ch.codePointAt(0);
+    if (ch === "\\" || code <= 0x20 || (code >= 0x7f && code <= 0x9f)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Return the host a git client connects to for a remote, read the way git,
+ * libcurl and OpenSSH read it rather than the way new URL() does. git ends the
+ * authority of ssh:// and git:// URLs only at the first slash, percent-decodes
+ * it before splitting off the host, and reads a leading "[...]" as the host.
+ * OpenSSH takes the host after the last "@". So a "?" or "#" before the first
+ * slash, a second "@", brackets in the user name, or a percent sign in an
+ * ssh:// or git:// authority make the host ambiguous.
+ *
+ * @param {string} remote Git URL or SCP-like address
+ * @returns {string|undefined} lower-cased host, or undefined when it is ambiguous
+ */
+export function getGitRemoteHost(remote) {
+  if (typeof remote !== "string") {
+    return undefined;
+  }
+  let authority;
+  const schemeEnd = remote.indexOf("://");
+  if (schemeEnd > -1) {
+    const scheme = remote.slice(0, schemeEnd).toLowerCase();
+    const rest = remote.slice(schemeEnd + 3);
+    const slash = rest.indexOf("/");
+    authority = slash > -1 ? rest.slice(0, slash) : rest;
+    // libcurl reads the http(s) authority as written. git decodes the others.
+    if (!["http", "https"].includes(scheme) && authority.includes("%")) {
+      return undefined;
+    }
+  } else {
+    // git reads user@host:path as SCP-like only when the colon comes before
+    // any slash. Otherwise it is a local path.
+    const colon = remote.indexOf(":");
+    const slash = remote.indexOf("/");
+    if (colon < 0 || (slash > -1 && slash < colon)) {
+      return undefined;
+    }
+    authority = remote.slice(0, colon);
+  }
+  if (authority.includes("?") || authority.includes("#")) {
+    return undefined;
+  }
+  const at = authority.indexOf("@");
+  if (at !== authority.lastIndexOf("@")) {
+    return undefined;
+  }
+  const userInfo = at > -1 ? authority.slice(0, at) : "";
+  if (userInfo.includes("[") || userInfo.includes("]")) {
+    return undefined;
+  }
+  let host = authority.slice(at + 1);
+  if (host.startsWith("[")) {
+    const close = host.indexOf("]");
+    if (close < 0) {
+      return undefined;
+    }
+    host = host.slice(0, close + 1);
+  } else {
+    const portSep = host.indexOf(":");
+    if (portSep > -1) {
+      host = host.slice(0, portSep);
+    }
+  }
+  if (!host || host.startsWith("-")) {
+    return undefined;
+  }
+  return host.toLowerCase();
+}
+
 /**
  * Clone a git repository into a temporary directory.
  *
@@ -606,6 +803,11 @@ export function validateAndRejectGitSource(filePath) {
  * @returns {string} cloned directory path
  */
 export function gitClone(repoUrl, branch = null) {
+  // Callers validate first, but git must never see a remote that fails it.
+  const validationError = validateAndRejectGitSource(repoUrl);
+  if (validationError) {
+    throw new Error(`${validationError.error}: ${validationError.details}`);
+  }
   let baseDirName = path.basename(repoUrl);
   if (!/^[a-zA-Z0-9_-]+$/.test(baseDirName)) {
     baseDirName = "repo-";
@@ -635,6 +837,8 @@ export function gitClone(repoUrl, branch = null) {
     "safe.bareRepository=explicit",
     "-c",
     "core.hooksPath=/dev/null",
+    "-c",
+    "core.symlinks=false",
     "clone",
     "--template=",
     repoUrl,
@@ -735,7 +939,7 @@ export function findGitRefForPurlVersion(repoUrl, purlResolution) {
       "--",
       repoUrl,
     ],
-    {},
+    { noRepository: true },
   );
   if (result.status !== 0 || !result.stdout) {
     return undefined;
@@ -1165,6 +1369,8 @@ export function resetGitUrlSourcePrefetch() {
   gitUrlSourcePrefetch = new Map();
 }
 
+registerRunReset("source:registry-documents", resetGitUrlSourcePrefetch);
+
 /**
  * Prefetch the registry documents a list of purls will be resolved from.
  *
@@ -1218,12 +1424,10 @@ async function fetchRegistryDocument(lookup) {
   // whose policy is one request per second however the request was reached.
   return (
     prefetchedResponse(gitUrlSourcePrefetch, lookup.url) ||
-    (await withHostRateLimit(lookup.url, () =>
-      cdxgenAgent.get(lookup.url, {
-        ...(lookup.accept ? { headers: { Accept: lookup.accept } } : {}),
-        responseType: "json",
-      }),
-    ))
+    (await gatedGet(lookup.url, {
+      ...(lookup.accept ? { headers: { Accept: lookup.accept } } : {}),
+      responseType: "json",
+    }))
   );
 }
 

package/lib/inventory/spdx.js

@@ -343,12 +343,257 @@ export function findLicenseId(name) {
     : name;
 }
 
+/**
+ * Identify a license from the title and opening of its full text. A license
+ * text quotes other licenses (the GNU GPL's text names the GNU Lesser GPL),
+ * so the substring scan of `guessLicenseId` alone can name the wrong one.
+ *
+ * @param {string} content License file contents
+ * @returns {string|undefined} SPDX license id or expression
+ */
+export function licenseIdFromText(content) {
+  if (typeof content !== "string" || !content.trim()) {
+    return undefined;
+  }
+  const head = content.slice(0, 4000).replace(/\s+/g, " ").trim().toUpperCase();
+  const version = (prefix) => {
+    const at = head.indexOf(prefix);
+    if (at < 0) {
+      return undefined;
+    }
+    const tail = head.slice(at + prefix.length, at + prefix.length + 60);
+    const match = tail.match(/VERSION (\d(?:\.\d)?)/);
+    return match?.[1];
+  };
+  // the first lines name the license; quotations of others come later, so
+  // of the GNU licenses the one named first is the one the text is
+  const title = head.slice(0, 200);
+  const gnu = [
+    ["GNU AFFERO GENERAL PUBLIC LICENSE", { 3: "AGPL-3.0-only" }],
+    [
+      "GNU LESSER GENERAL PUBLIC LICENSE",
+      { 3: "LGPL-3.0-only", 2.1: "LGPL-2.1-only", 2: "LGPL-2.0-only" },
+    ],
+    [
+      "GNU LIBRARY GENERAL PUBLIC LICENSE",
+      { 2.1: "LGPL-2.1-only", 2: "LGPL-2.0-only" },
+    ],
+    [
+      "GNU GENERAL PUBLIC LICENSE",
+      { 3: "GPL-3.0-only", 2: "GPL-2.0-only", 1: "GPL-1.0-only" },
+    ],
+  ]
+    .map(([name, ids]) => ({ name, ids, at: title.indexOf(name) }))
+    .filter((t) => t.at >= 0)
+    .sort((a, b) => a.at - b.at)[0];
+  if (gnu) {
+    return gnu.ids[version(gnu.name)];
+  }
+  if (head.includes("APACHE LICENSE") && head.includes("VERSION 2.0")) {
+    return head.includes("LLVM EXCEPTION")
+      ? "Apache-2.0 WITH LLVM-exception"
+      : "Apache-2.0";
+  }
+  if (
+    title.includes("MOZILLA PUBLIC LICENSE") &&
+    head.includes("VERSION 2.0")
+  ) {
+    return "MPL-2.0";
+  }
+  if (title.includes("BOOST SOFTWARE LICENSE - VERSION 1.0")) {
+    return "BSL-1.0";
+  }
+  if (
+    head.includes(
+      "THIS IS FREE AND UNENCUMBERED SOFTWARE RELEASED INTO THE PUBLIC DOMAIN",
+    )
+  ) {
+    return "Unlicense";
+  }
+  if (head.includes("PERMISSION IS HEREBY GRANTED, FREE OF CHARGE")) {
+    return "MIT";
+  }
+  if (
+    head.includes(
+      "PERMISSION TO USE, COPY, MODIFY, AND/OR DISTRIBUTE THIS SOFTWARE FOR ANY PURPOSE WITH OR WITHOUT FEE IS HEREBY GRANTED",
+    )
+  ) {
+    return "ISC";
+  }
+  if (
+    head.includes("THIS SOFTWARE IS PROVIDED 'AS-IS'") &&
+    head.includes(
+      "PERMISSION IS GRANTED TO ANYONE TO USE THIS SOFTWARE FOR ANY PURPOSE",
+    )
+  ) {
+    return "Zlib";
+  }
+  if (head.includes("REDISTRIBUTION AND USE IN SOURCE AND BINARY FORMS")) {
+    if (head.includes("ALL ADVERTISING MATERIALS MENTIONING")) {
+      return "BSD-4-Clause";
+    }
+    return head.includes("NEITHER THE NAME") ||
+      head.includes("THE NAME OF THE AUTHOR MAY NOT BE USED")
+      ? "BSD-3-Clause"
+      : "BSD-2-Clause";
+  }
+  return undefined;
+}
+
+// The GNU licenses by the title line their text opens with, and their SPDX
+// ids by version.
+const GNU_LICENSE_TITLES = {
+  "GNU AFFERO GENERAL PUBLIC LICENSE": { 3: "AGPL-3.0-only" },
+  "GNU LESSER GENERAL PUBLIC LICENSE": {
+    3: "LGPL-3.0-only",
+    2.1: "LGPL-2.1-only",
+    2: "LGPL-2.0-only",
+  },
+  "GNU LIBRARY GENERAL PUBLIC LICENSE": {
+    2.1: "LGPL-2.1-only",
+    2: "LGPL-2.0-only",
+  },
+  "GNU GENERAL PUBLIC LICENSE": {
+    3: "GPL-3.0-only",
+    2: "GPL-2.0-only",
+    1: "GPL-1.0-only",
+  },
+};
+
+// Phrases that only the grant of one license carries, so finding one anywhere
+// in a notice means that license is granted, not merely mentioned.
+const LICENSE_GRANT_PHRASES = [
+  ["PERMISSION IS HEREBY GRANTED, FREE OF CHARGE", "MIT"],
+  [
+    "PERMISSION TO USE, COPY, MODIFY, AND/OR DISTRIBUTE THIS SOFTWARE FOR ANY PURPOSE WITH OR WITHOUT FEE IS HEREBY GRANTED",
+    "ISC",
+  ],
+  [
+    "PERMISSION IS GRANTED TO ANYONE TO USE THIS SOFTWARE FOR ANY PURPOSE",
+    "Zlib",
+  ],
+  [
+    "THIS IS FREE AND UNENCUMBERED SOFTWARE RELEASED INTO THE PUBLIC DOMAIN",
+    "Unlicense",
+  ],
+  ["BOOST SOFTWARE LICENSE - VERSION 1.0", "BSL-1.0"],
+  ["MOZILLA PUBLIC LICENSE VERSION 2.0", "MPL-2.0"],
+  ["MOZILLA PUBLIC LICENSE, VERSION 2.0", "MPL-2.0"],
+  ["MOZILLA PUBLIC LICENSE, V. 2.0", "MPL-2.0"],
+  [
+    "CREATIVE COMMONS ATTRIBUTION 4.0 INTERNATIONAL PUBLIC LICENSE",
+    "CC-BY-4.0",
+  ],
+  [
+    "CREATIVE COMMONS ATTRIBUTION-SHAREALIKE 4.0 INTERNATIONAL PUBLIC LICENSE",
+    "CC-BY-SA-4.0",
+  ],
+  ["CC0 1.0 UNIVERSAL", "CC0-1.0"],
+];
+
+/**
+ * Identify every license a notice grants, in the order the notice grants
+ * them. A notice can grant several: one LICENSE file holding an MIT grant and
+ * an Apache-2.0 grant (gopkg.in/yaml.v3), or a choice of licenses followed by
+ * each full text (github.com/spdx/tools-golang). A license text quotes others
+ * (the GNU GPL names the GNU Lesser GPL, the MPL names the GNU licenses), so
+ * only phrases that a grant carries count: a GNU license is found by its title
+ * standing on a line of its own, the others by a sentence of their grant.
+ * A notice none of these phrases match falls back to {@link guessLicenseId}.
+ *
+ * @param {string} content License file contents
+ * @returns {string[]} SPDX license ids or expressions; empty when none is
+ *   recognized
+ */
+export function guessLicenseIds(content) {
+  if (typeof content !== "string" || !content.trim()) {
+    return [];
+  }
+  const text = content.replace(/\s+/g, " ").toUpperCase();
+  const found = [];
+  const add = (at, id) => {
+    if (at >= 0 && id) {
+      found.push({ at, id });
+    }
+  };
+  // Walk the lines alongside `text`, so a title line is placed in it.
+  let offset = 0;
+  for (const line of content.split("\n")) {
+    const normalized = line.replace(/\s+/g, " ").trim().toUpperCase();
+    const at = normalized ? text.indexOf(normalized, offset) : -1;
+    if (at < 0) {
+      continue;
+    }
+    offset = at + normalized.length;
+    const ids = GNU_LICENSE_TITLES[normalized];
+    if (ids) {
+      const version = text
+        .slice(offset, offset + 60)
+        .match(/VERSION (\d(?:\.\d)?)/)?.[1];
+      add(at, ids[version]);
+    }
+  }
+  for (const marker of [
+    "APACHE LICENSE VERSION 2.0",
+    "APACHE LICENSE, VERSION 2.0",
+    "APACHE LICENSE 2.0",
+  ]) {
+    add(
+      text.indexOf(marker),
+      text.includes("LLVM EXCEPTION")
+        ? "Apache-2.0 WITH LLVM-exception"
+        : "Apache-2.0",
+    );
+  }
+  for (const [phrase, id] of LICENSE_GRANT_PHRASES) {
+    add(text.indexOf(phrase), id);
+  }
+  // A notice can hold several BSD texts (one per copyright holder), each read
+  // up to where the next one starts.
+  const bsdPhrase = "REDISTRIBUTION AND USE IN SOURCE AND BINARY FORMS";
+  for (
+    let bsdAt = text.indexOf(bsdPhrase);
+    bsdAt >= 0;
+    bsdAt = text.indexOf(bsdPhrase, bsdAt + bsdPhrase.length)
+  ) {
+    const nextAt = text.indexOf(bsdPhrase, bsdAt + bsdPhrase.length);
+    const clauses = text.slice(
+      bsdAt,
+      Math.min(bsdAt + 2000, nextAt < 0 ? text.length : nextAt),
+    );
+    add(
+      bsdAt,
+      clauses.includes("ALL ADVERTISING MATERIALS MENTIONING")
+        ? "BSD-4-Clause"
+        : clauses.includes("NEITHER THE NAME") ||
+            clauses.includes("THE NAME OF THE AUTHOR MAY NOT BE USED")
+          ? "BSD-3-Clause"
+          : "BSD-2-Clause",
+    );
+  }
+  const ids = [];
+  for (const { id } of found.sort((a, b) => a.at - b.at)) {
+    if (!ids.includes(id)) {
+      ids.push(id);
+    }
+  }
+  if (ids.length) {
+    return ids;
+  }
+  const guessed = guessLicenseId(content);
+  return guessed ? [guessed] : [];
+}
+
 /**
  * Method to guess the spdx license id from license contents
  *
  * @param {string} content License file contents
  */
 export function guessLicenseId(content) {
+  const fromText = licenseIdFromText(content);
+  if (fromText) {
+    return fromText;
+  }
   content = content.replace(/\n/g, " ");
   for (const l of licenseMapping) {
     for (const j in l.names) {