@cdxgen/cdxgen 13.2.0 → 13.4.0

package/lib/ecosystems/cargoCache.js

@@ -1,11 +1,14 @@
 import { readdirSync, readFileSync, statSync } from "node:fs";
 import { homedir } from "node:os";
-import { join, resolve } from "node:path";
+import { join, posix, resolve } from "node:path";
+import { gunzipSync } from "node:zlib";
 
-import * as toml from "smol-toml";
+import { Parser as TarParser } from "tar";
 
 import { readEnvironmentVariable } from "../core/activity.js";
 import { safeExistsSync } from "../core/fs.js";
+import { registerRunReset } from "../core/runState.js";
+import { parseToml } from "../core/toml.js";
 
 /**
  * Crate metadata read from the local Cargo registry, without any network.
@@ -69,6 +72,8 @@ export function resetCargoCacheState() {
   _registryDirs = undefined;
 }
 
+registerRunReset("cargo:local-registry", resetCargoCacheState);
+
 /**
  * Whether the local registry should be consulted before crates.io.
  *
@@ -101,15 +106,16 @@ export function getCargoRegistryDir() {
  * private registry has more than one, and the crate being looked up may live
  * in any of them, so all are searched.
  *
- * @returns {{src: string[], index: string[]}} Absolute directory paths.
+ * @returns {{src: string[], index: string[], cache: string[]}} Absolute
+ *   directory paths.
  */
 function registryDirs() {
   if (_registryDirs) {
     return _registryDirs;
   }
   const root = getCargoRegistryDir();
-  _registryDirs = { src: [], index: [] };
-  for (const kind of ["src", "index"]) {
+  _registryDirs = { src: [], index: [], cache: [] };
+  for (const kind of ["src", "index", "cache"]) {
     const base = join(root, kind);
     try {
       for (const entry of readdirSync(base, { withFileTypes: true })) {
@@ -133,7 +139,7 @@ function registryDirs() {
  * @returns {CargoCacheMetadata|undefined} Metadata, or undefined when neither
  *   local source has this crate.
  */
-export function readCargoCacheMetadata(name, version) {
+export async function readCargoCacheMetadata(name, version) {
   if (!name || !version) {
     return undefined;
   }
@@ -144,6 +150,15 @@ export function readCargoCacheMetadata(name, version) {
     Object.assign(result, manifest.values);
     sources.push(manifest.source);
   }
+  const archive = await readCachedCrateArchive(name, version);
+  if (archive) {
+    for (const [key, value] of Object.entries(archive.values)) {
+      if (result[key] === undefined) {
+        result[key] = value;
+      }
+    }
+    sources.push(archive.source);
+  }
   const index = readIndexEntry(name, version);
   if (index) {
     // The manifest is authoritative for the fields both carry, since it is the
@@ -158,6 +173,198 @@ export function readCargoCacheMetadata(name, version) {
   return sources.length ? result : undefined;
 }
 
+/**
+ * A `.crate` archive larger than this is not opened for its manifest. Crates
+ * this size are rare, and the archive may be untrusted input.
+ */
+const CRATE_ARCHIVE_MAX_BYTES = 16 * 1024 * 1024;
+
+/**
+ * An archive that unpacks to more than this is not read, so a small archive
+ * that expands without bound costs bounded time and memory.
+ */
+const CRATE_UNPACKED_MAX_BYTES = 64 * 1024 * 1024;
+
+/**
+ * A single entry larger than this is not read out of a `.crate` archive.
+ */
+const CRATE_ENTRY_MAX_BYTES = 1024 * 1024;
+
+/**
+ * The `license-file` of a manifest as a path inside the crate, or undefined
+ * when it points anywhere else. The manifest is the crate author's, so a path
+ * that climbs out of the crate must never be followed.
+ *
+ * @param {*} licenseFile The manifest's `license-file` value
+ * @returns {string|undefined} Normalised relative path
+ */
+function crateLicenseFilePath(licenseFile) {
+  if (typeof licenseFile !== "string" || !licenseFile.trim()) {
+    return undefined;
+  }
+  const normalized = posix.normalize(licenseFile.trim().replaceAll("\\", "/"));
+  if (
+    normalized.startsWith("/") ||
+    normalized === ".." ||
+    normalized.startsWith("../") ||
+    /^[A-Za-z]:/.test(normalized)
+  ) {
+    return undefined;
+  }
+  return normalized;
+}
+
+/**
+ * The unpacked tar data of a `.crate` archive, within the size bounds.
+ *
+ * @param {string} crateFile Path to the `.crate` archive (a gzipped tar)
+ * @returns {Buffer|undefined}
+ */
+function unpackCrateArchive(crateFile) {
+  try {
+    if (statSync(crateFile).size > CRATE_ARCHIVE_MAX_BYTES) {
+      return undefined;
+    }
+    return gunzipSync(readFileSync(crateFile), {
+      maxOutputLength: CRATE_UNPACKED_MAX_BYTES,
+    });
+  } catch {
+    // An unreadable archive, or one that unpacks past the bound, is the same
+    // as none: the network path answers.
+    return undefined;
+  }
+}
+
+/**
+ * Read one file out of unpacked `.crate` tar data.
+ *
+ * @param {Buffer} tarData Unpacked tar data
+ * @param {string} entryName Entry path relative to the archive root
+ * @returns {Promise<string|undefined>} The entry's text, or undefined
+ */
+function readCrateArchiveEntry(tarData, entryName) {
+  return new Promise((resolvePromise) => {
+    let content;
+    const parser = new TarParser({
+      onReadEntry: (entry) => {
+        const path = entry.path.startsWith("./")
+          ? entry.path.slice(2)
+          : entry.path;
+        if (
+          content !== undefined ||
+          path !== entryName ||
+          entry.size > CRATE_ENTRY_MAX_BYTES
+        ) {
+          entry.resume();
+          return;
+        }
+        const chunks = [];
+        entry.on("data", (chunk) => chunks.push(chunk));
+        entry.on("end", () => {
+          content = Buffer.concat(chunks).toString("utf-8");
+        });
+      },
+    });
+    parser.on("error", () => resolvePromise(undefined));
+    parser.on("close", () => resolvePromise(content));
+    parser.end(tarData);
+  });
+}
+
+/**
+ * The manifest values of a `.crate` archive, read without extracting it.
+ *
+ * @param {string} crateFile Path to the `.crate` archive
+ * @param {string} name Crate name, for the archive's inner directory
+ * @param {string} version Crate version
+ * @returns {Promise<{values: Object, source: string}|undefined>}
+ */
+export async function readCrateArchiveValues(crateFile, name, version) {
+  if (!name || !version) {
+    return undefined;
+  }
+  const tarData = unpackCrateArchive(crateFile);
+  if (!tarData) {
+    return undefined;
+  }
+  const prefix = `${name}-${version}`;
+  const manifestData = await readCrateArchiveEntry(
+    tarData,
+    `${prefix}/Cargo.toml`,
+  );
+  if (!manifestData) {
+    return undefined;
+  }
+  let parsed;
+  try {
+    parsed = parseToml(manifestData);
+  } catch {
+    return undefined;
+  }
+  const pkg = parsed?.package;
+  if (!pkg || typeof pkg !== "object") {
+    return undefined;
+  }
+  const values = {};
+  assignString(values, "description", pkg.description);
+  assignString(values, "repository", pkg.repository);
+  assignString(values, "homepage", pkg.homepage);
+  assignString(values, "rustVersion", pkg["rust-version"]);
+  assignString(values, "license", pkg.license);
+  const licenseFile = crateLicenseFilePath(pkg["license-file"]);
+  if (licenseFile && !values.license) {
+    assignLicenseText(
+      values,
+      await readCrateArchiveEntry(tarData, `${prefix}/${licenseFile}`),
+    );
+  }
+  return { values, source: crateFile };
+}
+
+/**
+ * The `.crate` archive of a crate version held in the registry cache, which is
+ * the only local copy of a crate the build downloaded but never compiled.
+ *
+ * @param {string} name Crate name.
+ * @param {string} version Exact version.
+ * @returns {Promise<{values: Object, source: string}|undefined>}
+ */
+async function readCachedCrateArchive(name, version) {
+  for (const dir of registryDirs().cache) {
+    const crateFile = join(dir, `${name}-${version}.crate`);
+    if (!safeExistsSync(crateFile)) {
+      continue;
+    }
+    const archive = await readCrateArchiveValues(crateFile, name, version);
+    if (archive) {
+      return archive;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Record a `license-file` notice as a component licence, following the shape
+ * cdxgen records licence file text in elsewhere: a CUSTOM licence name with
+ * the notice as its text.
+ *
+ * @param {Object} values Manifest values so far, updated in place
+ * @param {string|undefined} text The notice text
+ */
+function assignLicenseText(values, text) {
+  if (!text?.length) {
+    return;
+  }
+  values.licenses = [
+    {
+      license: {
+        name: "CUSTOM",
+        text: { contentType: "text/plain", content: text },
+      },
+    },
+  ];
+}
+
 /**
  * Read the normalised `Cargo.toml` of an extracted crate.
  *
@@ -173,7 +380,7 @@ function readExtractedManifest(name, version) {
     }
     let parsed;
     try {
-      parsed = toml.parse(readFileSync(manifestFile, "utf-8"));
+      parsed = parseToml(readFileSync(manifestFile, "utf-8"));
     } catch {
       // A manifest that does not parse is not worth a warning: the network
       // path still has the answer.
@@ -189,8 +396,24 @@ function readExtractedManifest(name, version) {
     assignString(values, "homepage", pkg.homepage);
     assignString(values, "rustVersion", pkg["rust-version"]);
     // A crate may carry `license` or, for a non-SPDX license, `license-file`.
-    // Only the expression is usable as a component license.
+    // The expression is used as the licence; a licence file is recorded as a
+    // CUSTOM licence carrying the notice text, the way cdxgen records licence
+    // file text elsewhere.
     assignString(values, "license", pkg.license);
+    const licenseFile = crateLicenseFilePath(pkg["license-file"]);
+    if (licenseFile && !values.license) {
+      try {
+        assignLicenseText(
+          values,
+          readFileSync(join(dir, `${name}-${version}`, licenseFile), {
+            encoding: "utf-8",
+          }),
+        );
+      } catch {
+        // A licence file that cannot be read leaves the licence to the network
+        // path, exactly as an absent one does.
+      }
+    }
     if (parsed.features && typeof parsed.features === "object") {
       values.features = parsed.features;
     }