@cdxgen/cdxgen 13.3.0 → 13.4.0

package/lib/ecosystems/parsers-elm.js

@@ -5,13 +5,10 @@ import process from "node:process";
 
 import semver from "semver";
 
-import {
-  cdxgenAgent,
-  DEBUG_MODE,
-  readEnvironmentVariable,
-} from "../core/activity.js";
+import { DEBUG_MODE, readEnvironmentVariable } from "../core/activity.js";
 import { safeExistsSync } from "../core/fs.js";
 import {
+  gatedGet,
   prefetchEnabled,
   prefetchedResponse,
   prefetchJson,
@@ -689,13 +686,28 @@ export function isRegistryCrawlable(name) {
  */
 export async function getElmMetadata(pkgList, dependencies) {
   const registry = elmRegistryUrl();
+  // The two documents are asked for separately, each only when a field it
+  // supplies is still missing after the local cache: the manifest carries
+  // the licence, the summary and the package's own dependencies, the
+  // endpoint the distribution URL and its hash. Neither carries
+  // provenance, so a provenance fetch, as under --bom-audit, asks for
+  // nothing more.
+  const refsWithoutEdges = new Set(
+    (dependencies || [])
+      .filter((edge) => !edge.dependsOn?.length)
+      .map((edge) => edge.ref),
+  );
+  const needsManifest = (pkg) =>
+    !pkg.license || !pkg.description || refsWithoutEdges.has(pkg["bom-ref"]);
+  const needsEndpoint = (pkg) => !pkg.distribution?.url || !pkg.hashes?.length;
   const targets = pkgList.filter(
     (pkg) =>
       pkg.group &&
       pkg.version &&
       semver.valid(pkg.version) &&
       splitElmName(`${pkg.group}/${pkg.name}`) &&
-      isRegistryCrawlable(`${pkg.group}/${pkg.name}`),
+      isRegistryCrawlable(`${pkg.group}/${pkg.name}`) &&
+      (needsManifest(pkg) || needsEndpoint(pkg)),
   );
   if (!targets.length) {
     return pkgList;
@@ -706,10 +718,12 @@ export async function getElmMetadata(pkgList, dependencies) {
     `${registry}/packages/${pkg.group}/${pkg.name}/${pkg.version}/endpoint.json`;
   const prefetched = await prefetchJson(
     prefetchEnabled()
-      ? targets.flatMap((pkg) => [
-          { url: manifestUrl(pkg) },
-          { url: endpointUrl(pkg) },
-        ])
+      ? targets
+          .flatMap((pkg) => [
+            needsManifest(pkg) ? { url: manifestUrl(pkg) } : undefined,
+            needsEndpoint(pkg) ? { url: endpointUrl(pkg) } : undefined,
+          ])
+          .filter(Boolean)
       : [],
   );
   const manifests = new Map();
@@ -717,25 +731,29 @@ export async function getElmMetadata(pkgList, dependencies) {
     if (DEBUG_MODE) {
       console.log(`Querying ${registry} for ${pkg.group}/${pkg.name}`);
     }
-    const manifest = await fetchJson(manifestUrl(pkg), prefetched);
-    if (manifest) {
-      manifests.set(pkg["bom-ref"], manifest);
-      if (!pkg.license && typeof manifest.license === "string") {
-        pkg.license = manifest.license;
-      }
-      if (!pkg.description && typeof manifest.summary === "string") {
-        pkg.description = manifest.summary;
+    if (needsManifest(pkg)) {
+      const manifest = await fetchJson(manifestUrl(pkg), prefetched);
+      if (manifest) {
+        manifests.set(pkg["bom-ref"], manifest);
+        if (!pkg.license && typeof manifest.license === "string") {
+          pkg.license = manifest.license;
+        }
+        if (!pkg.description && typeof manifest.summary === "string") {
+          pkg.description = manifest.summary;
+        }
       }
     }
-    const endpoint = await fetchJson(endpointUrl(pkg), prefetched);
-    if (typeof endpoint?.url === "string") {
-      pkg.distribution = { url: endpoint.url };
-    }
-    if (/^[a-f0-9]{40}$/i.test(endpoint?.hash || "")) {
-      // The compiler streams the archive and compares this digest against the
-      // SHA-1 of the bytes it received, so it is a content hash of the
-      // distribution rather than a revision identifier.
-      pkg.hashes = [{ alg: "SHA-1", content: endpoint.hash.toLowerCase() }];
+    if (needsEndpoint(pkg)) {
+      const endpoint = await fetchJson(endpointUrl(pkg), prefetched);
+      if (typeof endpoint?.url === "string") {
+        pkg.distribution = { url: endpoint.url };
+      }
+      if (/^[a-f0-9]{40}$/i.test(endpoint?.hash || "")) {
+        // The compiler streams the archive and compares this digest against the
+        // SHA-1 of the bytes it received, so it is a content hash of the
+        // distribution rather than a revision identifier.
+        pkg.hashes = [{ alg: "SHA-1", content: endpoint.hash.toLowerCase() }];
+      }
     }
   }
   if (dependencies?.length && manifests.size) {
@@ -790,7 +808,7 @@ async function fetchJson(url, prefetched) {
   try {
     const res =
       prefetchedResponse(prefetched, url) ||
-      (await cdxgenAgent.get(url, { responseType: "json" }));
+      (await gatedGet(url, { responseType: "json" }));
     return res?.body;
   } catch (err) {
     if (DEBUG_MODE) {

package/lib/ecosystems/parsers-gleam.js

@@ -1,7 +1,6 @@
 import { readFileSync } from "node:fs";
 
-import * as toml from "smol-toml";
-
+import { parseToml } from "../core/toml.js";
 import { tryBuildPurl } from "../inventory/purl.js";
 
 /**
@@ -35,7 +34,7 @@ import { tryBuildPurl } from "../inventory/purl.js";
 export function parseGleamProject(gleamTomlFile, manifestTomlFile) {
   let manifest;
   try {
-    manifest = toml.parse(readFileSync(gleamTomlFile, "utf-8"));
+    manifest = parseToml(readFileSync(gleamTomlFile, "utf-8"));
   } catch (error) {
     console.warn(`Failed to parse ${gleamTomlFile}: ${error.message}`);
     return { pkgList: [], dependencies: [], parentComponent: {} };
@@ -70,7 +69,7 @@ export function parseGleamProject(gleamTomlFile, manifestTomlFile) {
   let lock;
   if (manifestTomlFile) {
     try {
-      lock = toml.parse(readFileSync(manifestTomlFile, "utf-8"));
+      lock = parseToml(readFileSync(manifestTomlFile, "utf-8"));
     } catch (error) {
       console.warn(`Failed to parse ${manifestTomlFile}: ${error.message}`);
       lock = undefined;