@cdxgen/cdxgen 13.2.0 → 13.4.0

package/lib/core/fs.js

@@ -3,6 +3,7 @@ import { spawnSync } from "node:child_process";
 import { createHash, randomUUID } from "node:crypto";
 import {
   accessSync,
+  closeSync,
   copyFileSync,
   createReadStream,
   existsSync,
@@ -10,10 +11,14 @@ import {
   lstatSync,
   mkdirSync,
   mkdtempSync,
+  openSync,
   readFileSync,
+  renameSync,
   rmSync,
+  statSync,
   unlinkSync,
   writeFileSync,
+  writeSync,
 } from "node:fs";
 import { tmpdir } from "node:os";
 import path, { basename, dirname, join, resolve } from "node:path";
@@ -144,6 +149,53 @@ export function safeIsExecutableSync(filePath) {
  * @returns {void}
  */
 export function safeWriteSync(filePath, data, options) {
+  if (isWriteBlocked(filePath)) {
+    return undefined;
+  }
+  writeFileSync(filePath, data, options);
+  recordFilesystemActivity("write", filePath, "completed");
+}
+
+/**
+ * Permission- and dry-run-aware write of a sequence of chunks, for content too
+ * large to join into one string first. The chunks go to a temporary file next
+ * to the target, which is renamed into place once complete, so a chunk may be
+ * read from the file being replaced and a failed write leaves the target as it
+ * was.
+ *
+ * @param {string} filePath File path to write.
+ * @param {Iterable<string|Buffer>} chunks Chunks written in order.
+ * @returns {void}
+ */
+export function safeWriteChunksSync(filePath, chunks) {
+  if (isWriteBlocked(filePath)) {
+    return undefined;
+  }
+  const tempPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`;
+  const fd = openSync(tempPath, "w");
+  try {
+    for (const chunk of chunks) {
+      const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
+      let written = 0;
+      while (written < bytes.length) {
+        written += writeSync(fd, bytes, written);
+      }
+    }
+    closeSync(fd);
+    renameSync(tempPath, filePath);
+  } catch (err) {
+    try {
+      closeSync(fd);
+    } catch {
+      // Already closed before the rename failed.
+    }
+    rmSync(tempPath, { force: true });
+    throw err;
+  }
+  recordFilesystemActivity("write", filePath, "completed");
+}
+
+function isWriteBlocked(filePath) {
   if (isDryRun) {
     recordFilesystemActivity(
       "write",
@@ -151,7 +203,7 @@ export function safeWriteSync(filePath, data, options) {
       "blocked",
       "Dry run mode blocks filesystem writes.",
     );
-    return undefined;
+    return true;
   }
   if (!hasWritePermission(filePath)) {
     if (DEBUG_MODE) {
@@ -163,10 +215,9 @@ export function safeWriteSync(filePath, data, options) {
       "blocked",
       "cdxgen lacks write permission for this path.",
     );
-    return undefined;
+    return true;
   }
-  writeFileSync(filePath, data, options);
-  recordFilesystemActivity("write", filePath, "completed");
+  return false;
 }
 
 /**
@@ -483,6 +534,116 @@ const VERSION_PROBE_ARGS = new Set(["--version", "-version", "version"]);
 const POSIX_SHELL_METACHARACTERS = /[;&|<>$`\\\n\r]/;
 const WINDOWS_SHELL_METACHARACTERS = /[&|<>^%\n\r]/;
 
+/**
+ * Quote one argument of a shell command line that is built as a single
+ * string. The shell flag exists so cmd.exe can resolve bare command names,
+ * `.cmd` shims and scripts, and every argument is data (paths, flags) that
+ * must arrive as one token whatever it contains: joined unquoted, an
+ * argument with spaces splits into several and hands the tool the wrong
+ * tokens.
+ *
+ * Windows follows the quoting rules cmd.exe and the MSVC runtimes share:
+ * wrap in double quotes, doubling the backslashes that precede a quote and
+ * escaping the quote itself. Inside double quotes cmd.exe already treats
+ * `&`, `|`, `<`, `>`, `^` and parentheses as literals, and the
+ * shell-metacharacter block above refuses `%` and the rest before this
+ * runs. A POSIX shell gets single quotes, which no expansion can leave.
+ *
+ * The same quoting is applied to the command itself when it names an
+ * existing file under a path with spaces; a command the shell must parse
+ * (a bare name, `sh -c`) never names a file as a whole and stays verbatim.
+ *
+ * @param {string} arg Argument to quote
+ * @param {boolean} [forWindows] Quote for cmd.exe rather than a POSIX shell; defaults to the current platform
+ * @returns {string} The argument, quoted when the shell needs it
+ */
+export function shellQuoteArgument(arg, forWindows = isWin) {
+  const value = `${arg}`;
+  if (forWindows) {
+    if (value === "") {
+      return '""';
+    }
+    if (!/[\s"]/.test(value)) {
+      return value;
+    }
+    // Every backslash run that precedes a quote doubles, each quote escapes,
+    // and the trailing run doubles for the closing quote this returns. A
+    // character scan keeps it linear; the equivalent `(\\*)"` pattern is
+    // polynomial on runs of backslashes that no quote ends.
+    let quoted = "";
+    let backslashes = 0;
+    for (const ch of value) {
+      if (ch === "\\") {
+        backslashes++;
+        quoted += ch;
+      } else if (ch === '"') {
+        quoted += `${"\\".repeat(backslashes)}\\"`;
+        backslashes = 0;
+      } else {
+        quoted += ch;
+        backslashes = 0;
+      }
+    }
+    quoted += "\\".repeat(backslashes);
+    return `"${quoted}"`;
+  }
+  if (value === "") {
+    return "''";
+  }
+  if (!/[^-\w@%+=:,./]/.test(value)) {
+    return value;
+  }
+  return `'${value.replaceAll("'", `'\\''`)}'`;
+}
+
+/**
+ * Whether the path names an existing regular file, without throwing on the
+ * many values that are not paths at all (a bare command name, a command with
+ * arguments, anything the shell is meant to parse).
+ *
+ * @param {string} file Candidate path
+ * @returns {boolean} True when a regular file exists at the path
+ */
+function isFileNoThrow(file) {
+  try {
+    return statSync(`${file}`).isFile();
+  } catch (_err) {
+    return false;
+  }
+}
+
+/**
+ * Build the single command string a shell spawn runs. Every argument is
+ * quoted for the platform shell, so an argument containing spaces stays one
+ * token and cannot alter the command, and a command that is itself a file
+ * under a path with spaces (`C:\Program Files\...\node.exe`) is quoted too;
+ * a command the shell must parse (`sh -c`, a bare name) never names an
+ * existing file as a whole and is left verbatim.
+ *
+ * The one exception is a caller that composes shell fragments itself (sbt
+ * command lines embed their own quoting for the shell to strip): it passes
+ * `rawShellArgs` and keeps the verbatim join, under the shell-metacharacter
+ * block in safeSpawnSync like every other argument.
+ *
+ * @param {string} command The executable
+ * @param {string[]|undefined} args Arguments, when the caller passed any
+ * @param {boolean} [rawShellArgs] Join the arguments verbatim instead of quoting them
+ * @returns {string} The command line the shell will run
+ */
+export function joinShellCommand(command, args, rawShellArgs = false) {
+  let commandToken = `${command}`;
+  if (/\s/.test(commandToken) && isFileNoThrow(commandToken)) {
+    commandToken = shellQuoteArgument(commandToken);
+  }
+  if (!args?.length) {
+    return commandToken;
+  }
+  return [
+    commandToken,
+    ...(rawShellArgs ? args : args.map((arg) => shellQuoteArgument(arg))),
+  ].join(" ");
+}
+
 function hasShellMetacharacters(value) {
   if (value === undefined || value === null) {
     return false;
@@ -696,9 +857,12 @@ export function safeSpawnSync(command, args, options) {
       }
     }
   }
+  // Read before the option is stripped below: a caller composing shell
+  // fragments itself opts out of argument quoting with this flag.
+  const rawShellArgs = options?.cdxgenRawShellArgs === true;
   if (!options) {
     options = {};
-  } else if (options.cdxgenActivity) {
+  } else if (options.cdxgenActivity || options.cdxgenRawShellArgs) {
     options = {
       ...options,
     };
@@ -706,6 +870,9 @@ export function safeSpawnSync(command, args, options) {
   if (options.cdxgenActivity) {
     delete options.cdxgenActivity;
   }
+  if (options.cdxgenRawShellArgs) {
+    delete options.cdxgenRawShellArgs;
+  }
   if (options.shell === true) {
     const unsafeShellToken = getUnsafeShellToken(command, args);
     if (unsafeShellToken !== undefined) {
@@ -817,8 +984,9 @@ export function safeSpawnSync(command, args, options) {
   commandsExecuted.add(command);
   // Fix for DEP0190 warning
   if (options?.shell === true) {
+    const joined = joinShellCommand(command, args, rawShellArgs);
+    command = joined;
     if (args?.length) {
-      command = `${command} ${args.join(" ")}`;
       args = undefined;
     }
   }
@@ -857,7 +1025,8 @@ export const MAX_BUFFER =
 export function getAllFiles(dirPath, pattern, options = {}) {
   let ignoreList = [];
   if (!options.noIgnore) {
-    ignoreList.push("**/.hg/**", "**/.git/**");
+    // .chen holds chen's AST cache, which atom writes into a C/C++ source tree
+    ignoreList.push("**/.hg/**", "**/.git/**", "**/.chen/**");
     // Only ignore node_modules if the caller is not looking for package.json
     if (!pattern.includes("package.json") && !options.includeNodeModulesDir) {
       ignoreList.push("**/node_modules/**");

package/lib/core/httpClient.js

@@ -1,4 +1,5 @@
 import { Buffer } from "node:buffer";
+import { createHash } from "node:crypto";
 import process from "node:process";
 import { brotliDecompressSync, gunzipSync, inflateSync } from "node:zlib";
 
@@ -9,6 +10,8 @@ import {
   request as undiciRequest,
 } from "undici";
 
+import { registerRunReset } from "./runState.js";
+
 // Environment variables here are read with raw `process.env` rather than
 // `readEnvironmentVariable`: that helper lives in activity.js, which imports
 // this module, so routing these through it would form an import cycle.
@@ -41,9 +44,8 @@ import {
  *   `Content-Encoding` header, matching `got`'s transparent decoding.
  * - Proxy support via the standard `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY`
  *   environment variables (through undici's `EnvHttpProxyAgent`).
- * - An in-memory GET response cache (enabled by default, disabled by setting
- *   the `CDXGEN_NO_CACHE` environment variable) that replaces the got + Keyv
- *   cache cdxgen previously relied on.
+ * - An in-memory GET response cache that replaces the got + Keyv cache cdxgen
+ *   previously relied on. A request opts out with `cache: false`.
  * - Response objects exposing `statusCode`, `headers`, `body`, `rawBody`,
  *   `url` and `request.options`.
  * - A lazily-resolved `.json()` method on the returned promise, mirroring
@@ -137,7 +139,12 @@ function mergeHeaders(base = {}, override = {}) {
  *   `beforeError` arrays.
  */
 function mergeHooks(base = {}, override = {}) {
-  const hookNames = ["beforeRequest", "afterResponse", "beforeError"];
+  const hookNames = [
+    "beforeRequest",
+    "beforeNetwork",
+    "afterResponse",
+    "beforeError",
+  ];
   const merged = {};
   for (const name of hookNames) {
     merged[name] = [...(base?.[name] || []), ...(override?.[name] || [])];
@@ -253,6 +260,11 @@ function decompressBody(rawBody, contentEncoding) {
  * Read and decode an undici response body according to the requested
  * `responseType`.
  *
+ * An error response often carries a body in another format than the one asked
+ * for, such as the HTML page a CDN sends with a 429. Its JSON body is decoded
+ * when it parses and kept as text otherwise, so the caller sees the status
+ * rather than a parse error.
+ *
  * @param {Object} undiciResponse Response returned by `undici.request`.
  * @param {string} [responseType] One of `"json"`, `"buffer"` or `"text"`.
  * @returns {Promise<{body: any, rawBody: Buffer}>} Decoded body and raw bytes.
@@ -272,7 +284,17 @@ async function readBody(undiciResponse, responseType) {
     body = rawBody;
   } else if (responseType === "json") {
     const text = rawBody.toString("utf-8");
-    body = text.length ? JSON.parse(text) : undefined;
+    if (!text.length) {
+      body = undefined;
+    } else if (undiciResponse.statusCode >= 400) {
+      try {
+        body = JSON.parse(text);
+      } catch (_err) {
+        body = text;
+      }
+    } else {
+      body = JSON.parse(text);
+    }
   } else {
     body = rawBody.toString("utf-8");
   }
@@ -304,20 +326,100 @@ function resolveRequestBody(options) {
 
 // In-memory cache of successful GET responses, mirroring the HTTP cache cdxgen
 // historically kept via got + Keyv. Repeated metadata/license lookups during a
-// scan are served from here instead of hitting the network again. Set the
-// CDXGEN_NO_CACHE environment variable to "true" or "1" to disable it.
+// scan are served from here instead of hitting the network again. It lasts for
+// one run; `CDXGEN_NO_CACHE` and `--no-cache` turn off only the on-disk caches,
+// because nothing gains from asking a registry the same question twice in one
+// scan.
+//
+// Entries expire after an hour and the cache is bounded in size, least
+// recently used first, so a long-running server neither serves stale registry
+// documents nor grows without limit.
 const responseCache = new Map();
+const RESPONSE_CACHE_TTL_MS = 60 * 60 * 1000;
+const RESPONSE_CACHE_MAX_BYTES = 512 * 1024 * 1024;
+let responseCacheLimits = {
+  ttlMs: RESPONSE_CACHE_TTL_MS,
+  maxBytes: RESPONSE_CACHE_MAX_BYTES,
+};
+let responseCacheBytes = 0;
+
+/**
+ * Approximate the memory a cached response holds. The raw bytes are kept,
+ * plus the decoded body for text and JSON responses.
+ *
+ * @param {Object} response Response object.
+ * @returns {number} Size in bytes.
+ */
+function cachedResponseBytes(response) {
+  const raw = response?.rawBody?.length || 0;
+  return response?.body === response?.rawBody ? raw : raw * 3;
+}
 
 /**
- * Determine whether the in-memory HTTP response cache is disabled via the
- * CDXGEN_NO_CACHE environment variable. Evaluated per request so tests and
- * callers can toggle it at runtime.
+ * Read a cached response, dropping it when it has expired and marking it as
+ * recently used otherwise.
  *
- * @returns {boolean} True when caching should be skipped.
+ * @param {string} key Cache key.
+ * @returns {Object|undefined} The cached response.
  */
-export function isCacheDisabled() {
-  const value = process.env.CDXGEN_NO_CACHE;
-  return value === "true" || value === "1";
+function cacheGet(key) {
+  const entry = responseCache.get(key);
+  if (!entry) {
+    return undefined;
+  }
+  responseCache.delete(key);
+  if (entry.expiresAt <= Date.now()) {
+    responseCacheBytes -= entry.bytes;
+    return undefined;
+  }
+  responseCache.set(key, entry);
+  return entry.response;
+}
+
+/**
+ * Store a response, evicting the least recently used entries beyond the size
+ * bound.
+ *
+ * @param {string} key Cache key.
+ * @param {Object} response Response object.
+ * @returns {void}
+ */
+function cacheSet(key, response) {
+  const bytes = cachedResponseBytes(response);
+  if (bytes > responseCacheLimits.maxBytes) {
+    return;
+  }
+  const previous = responseCache.get(key);
+  if (previous) {
+    responseCache.delete(key);
+    responseCacheBytes -= previous.bytes;
+  }
+  responseCache.set(key, {
+    response,
+    bytes,
+    expiresAt: Date.now() + responseCacheLimits.ttlMs,
+  });
+  responseCacheBytes += bytes;
+  for (const [oldestKey, oldest] of responseCache) {
+    if (responseCacheBytes <= responseCacheLimits.maxBytes) {
+      break;
+    }
+    responseCache.delete(oldestKey);
+    responseCacheBytes -= oldest.bytes;
+  }
+}
+
+/**
+ * Override the response cache bounds. Returns the previous bounds so tests can
+ * restore them. Not part of the public API.
+ *
+ * @param {{ttlMs?: number, maxBytes?: number}} limits New bounds.
+ * @returns {{ttlMs: number, maxBytes: number}} Previous bounds.
+ */
+export function _setHttpCacheLimits(limits) {
+  const previous = responseCacheLimits;
+  responseCacheLimits = { ...responseCacheLimits, ...limits };
+  return previous;
 }
 
 /**
@@ -327,8 +429,11 @@ export function isCacheDisabled() {
  */
 export function clearHttpCache() {
   responseCache.clear();
+  responseCacheBytes = 0;
 }
 
+registerRunReset("http:responses", clearHttpCache);
+
 // ---------------------------------------------------------------------------
 // Test cassette interceptor — the single network seam.
 //
@@ -397,15 +502,33 @@ export function clearHttpInterceptor() {
 }
 
 /**
- * Build the cache key for a request. Only the method and normalized URL are
- * used, matching got's default cache key behaviour.
+ * Build the cache key for a request.
+ *
+ * Besides the method and URL, the key carries the response type, which
+ * changes how the same bytes are decoded, and a digest of every request
+ * header. A response fetched with one credential is therefore never served to
+ * a request made with another credential or with none, and a JSON request
+ * never receives a text body. Header values only enter the key as a SHA-256
+ * digest, so no credential is held in it.
  *
  * @param {string} method Uppercased HTTP method.
  * @param {URL} url Request URL.
+ * @param {Object} [headers] Request headers as sent.
+ * @param {string} [responseType] Requested body decoding.
  * @returns {string} Cache key.
  */
-function cacheKeyFor(method, url) {
-  return `${method}:${url.toString()}`;
+function cacheKeyFor(method, url, headers = {}, responseType = undefined) {
+  const headerEntries = Object.entries(headers || {})
+    .filter(([, value]) => value !== undefined && value !== null)
+    .map(([name, value]) => [
+      name.toLowerCase(),
+      Array.isArray(value) ? value.join(", ") : String(value),
+    ])
+    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+  const headerDigest = headerEntries.length
+    ? createHash("sha256").update(JSON.stringify(headerEntries)).digest("hex")
+    : "";
+  return `${method}:${responseType || "text"}:${url.toString()}:${headerDigest}`;
 }
 
 // The maximum number of redirects to follow when `followRedirect` is enabled.
@@ -587,11 +710,23 @@ async function doRequest(mergedOptions) {
   const { body, headers } = resolveRequestBody(options);
   const method = (options.method || "GET").toUpperCase();
 
-  // Serve cacheable GET requests from the in-memory cache when enabled.
-  const cacheable = method === "GET" && !isCacheDisabled();
-  const cacheKey = cacheable ? cacheKeyFor(method, url) : undefined;
-  if (cacheable && responseCache.has(cacheKey)) {
-    return responseCache.get(cacheKey);
+  // Serve cacheable GET requests from the in-memory cache when enabled. A
+  // request opts out with `cache: false`, as token exchanges do.
+  const cacheable = method === "GET" && options.cache !== false;
+  const cacheKey = cacheable
+    ? cacheKeyFor(method, url, headers, options.responseType)
+    : undefined;
+  if (cacheable) {
+    const cached = cacheGet(cacheKey);
+    if (cached) {
+      return cached;
+    }
+  }
+
+  // Runs only when the request is about to reach the network, so a caller
+  // that rate-limits a host does not wait for a response already cached.
+  for (const hook of options.hooks?.beforeNetwork || []) {
+    await hook(options);
   }
 
   const timeoutMs = resolveTimeout(options.timeout);
@@ -652,7 +787,7 @@ async function doRequest(mergedOptions) {
 
   // Only successful responses are cached, matching HTTP cache semantics.
   if (cacheable && finalResponse.statusCode < 400) {
-    responseCache.set(cacheKey, finalResponse);
+    cacheSet(cacheKey, finalResponse);
   }
   return finalResponse;
 }

package/lib/core/metadataProperties.js

@@ -0,0 +1,97 @@
+/**
+ * The spellings under which a config file may list custom metadata
+ * properties: those of the `--metadata-property` flag, and the plural that
+ * the CDXGEN_METADATA_PROPERTIES environment variable maps to.
+ */
+export const METADATA_PROPERTY_CONFIG_KEYS = [
+  "metadata-property",
+  "metadataProperty",
+  "metadata-properties",
+  "metadataProperties",
+];
+
+// Namespaces whose properties cdxgen itself asserts: `cdx:` is its registered
+// taxonomy, and `internal:` carries its own bookkeeping
+const RESERVED_METADATA_PROPERTY_PREFIXES = ["cdx:", "internal:"];
+
+/**
+ * Report whether a property name belongs to a namespace cdxgen reserves for
+ * the facts it establishes itself, such as `cdx:bom:componentSrcFiles`.
+ *
+ * @param {string} name Property name
+ * @returns {boolean} true for a reserved name
+ */
+export function isReservedMetadataPropertyName(name) {
+  return RESERVED_METADATA_PROPERTY_PREFIXES.some((prefix) =>
+    `${name}`.startsWith(prefix),
+  );
+}
+
+/**
+ * Normalize user-supplied metadata properties into `[{name, value}]`
+ * (discussion 4391).
+ *
+ * Accepted shapes:
+ * - a `name=value` string, or a JSON document holding any of the shapes below
+ *   — an environment variable
+ * - an array of `name=value` strings or `{name, value}` objects — the
+ *   `--metadata-property` flag and a JSON config file
+ * - a plain object mapping a name to a value, or to a list of values for a
+ *   property that repeats — a JSON document or a YAML config file
+ *
+ * Values must be strings, numbers, or booleans. An entry without a name, a
+ * `name=value` string without a separator, and an empty or structured value
+ * are dropped rather than guessed at: a property with a mangled name or an
+ * `[object Object]` value is worse than none.
+ *
+ * @param {unknown} input Properties in any accepted shape
+ * @returns {{name: string, value: string}[]} Normalized properties
+ */
+export function parseMetadataProperties(input) {
+  const properties = [];
+  if (typeof input === "string") {
+    let parsed;
+    try {
+      parsed = JSON.parse(input);
+    } catch (_err) {
+      parsed = undefined;
+    }
+    input =
+      parsed && typeof parsed === "object"
+        ? parsed
+        : [typeof parsed === "string" ? parsed : input];
+  }
+  const append = (name, value) => {
+    if (typeof name !== "string" && typeof name !== "number") {
+      return;
+    }
+    const propName = `${name}`.trim();
+    if (
+      !propName.length ||
+      !["string", "number", "boolean"].includes(typeof value) ||
+      !`${value}`.length
+    ) {
+      return;
+    }
+    properties.push({ name: propName, value: `${value}` });
+  };
+  if (Array.isArray(input)) {
+    for (const entry of input) {
+      if (typeof entry === "string") {
+        const separator = entry.indexOf("=");
+        if (separator > 0) {
+          append(entry.slice(0, separator), entry.slice(separator + 1));
+        }
+      } else if (entry && typeof entry === "object") {
+        append(entry.name, entry.value);
+      }
+    }
+  } else if (input && typeof input === "object") {
+    for (const [name, value] of Object.entries(input)) {
+      for (const avalue of Array.isArray(value) ? value : [value]) {
+        append(name, avalue);
+      }
+    }
+  }
+  return properties;
+}