@cdxgen/cdxgen 13.3.0 → 13.5.0

package/lib/ecosystems/pythonutils.js

@@ -18,7 +18,7 @@ import {
   readEnvironmentVariable,
 } from "../core/activity.js";
 import { PYTHON_CMD, PYTHON_EXCLUDED_COMPONENTS } from "../core/env.js";
-import { safeExistsSync, safeSpawnSync } from "../core/fs.js";
+import { getAllFiles, safeExistsSync, safeSpawnSync } from "../core/fs.js";
 import { thoughtLog } from "../core/logger.js";
 import { isWin } from "../core/paths.js";
 import {
@@ -331,6 +331,70 @@ export function get_python_command_from_env(env) {
   return fallbackCmd;
 }
 
+/** Directories that hold installed packages, not the project's own code. */
+const INSTALLED_PACKAGE_DIRS = [
+  "**/site-packages/**",
+  "**/dist-packages/**",
+  "**/.venv/**",
+  "**/venv/**",
+  "**/.tox/**",
+  "**/.nox/**",
+  "**/__pypackages__/**",
+  "**/node_modules/**",
+];
+
+const projectModuleNamesCache = new Map();
+
+/**
+ * The top-level packages and modules a project provides, wherever they sit
+ * in its tree: each outermost package directory (one with an `__init__.py`
+ * whose parent has none), and the module files beside such packages, since
+ * their directory is a source root. Directories of installed packages
+ * (virtual environments, `site-packages`) are not the project's.
+ *
+ * @param {string} src Path to the project being analysed
+ * @returns {Set<string>} Module names
+ */
+export function projectModuleNames(src) {
+  const root = resolve(src);
+  if (projectModuleNamesCache.has(root)) {
+    return projectModuleNamesCache.get(root);
+  }
+  const names = new Set();
+  const sourceRoots = new Set();
+  const inits = getAllFiles(root, "**/__init__.py", {
+    exclude: INSTALLED_PACKAGE_DIRS,
+  });
+  for (const init of inits) {
+    let pkgDir = dirname(resolve(init));
+    while (
+      dirname(pkgDir) !== root &&
+      dirname(pkgDir).startsWith(root) &&
+      safeExistsSync(join(dirname(pkgDir), "__init__.py"))
+    ) {
+      pkgDir = dirname(pkgDir);
+    }
+    if (pkgDir === root) {
+      continue;
+    }
+    names.add(basename(pkgDir));
+    sourceRoots.add(dirname(pkgDir));
+  }
+  for (const dir of sourceRoots) {
+    try {
+      for (const f of readdirSync(dir)) {
+        if (f.endsWith(".py") && f !== "__init__.py") {
+          names.add(f.slice(0, -3));
+        }
+      }
+    } catch {
+      // an unreadable directory provides nothing
+    }
+  }
+  projectModuleNamesCache.set(root, names);
+  return names;
+}
+
 /**
  * Is this import satisfied by a module that lives in the project itself?
  *
@@ -344,7 +408,9 @@ export function get_python_command_from_env(env) {
  * filesystem whether the module is ours answers it exactly.
  *
  * Checks the project root and a `src/` layout for a module file, a package directory,
- * and a PEP 420 namespace package directory.
+ * and a PEP 420 namespace package directory, then the top-level packages found
+ * anywhere in the project and the modules beside them (tool-heavy repositories
+ * keep their Python libraries in places such as `dev_tools/pylibs/`).
  *
  * @param {string} src Path to the project being analysed
  * @param {string} name Imported top-level module name
@@ -354,6 +420,9 @@ export function isFirstPartyModule(src, name) {
   if (!src || !name || name.includes("/") || name.includes("\\")) {
     return false;
   }
+  if (projectModuleNames(src).has(name)) {
+    return true;
+  }
   for (const root of [src, join(src, "src")]) {
     if (
       safeExistsSync(join(root, `${name}.py`)) ||
@@ -378,6 +447,21 @@ export function isFirstPartyModule(src, name) {
   return false;
 }
 
+/**
+ * Whether a directory holds Python sources for atom to parse.
+ *
+ * The search skips the directories the manifest searches skip, such as
+ * node_modules and dot directories, and honours the exclude patterns, so a
+ * Python file shipped inside an npm package does not count.
+ *
+ * @param {string} src directory
+ * @param {Object} options CLI options
+ * @returns {boolean} true when the directory holds at least one Python file
+ */
+export function hasPythonSources(src, options = {}) {
+  return getAllFiles(src, "**/*.py", options).length > 0;
+}
+
 /**
  * Method to find python modules by parsing the imports and then checking with PyPI to obtain the latest version
  *
@@ -390,9 +474,11 @@ export async function getPyModules(src, epkgList, options) {
   const allImports = {};
   const dependenciesList = [];
   let modList = [];
-  const slicesFile = resolve(
-    options.depsSlicesFile || options.usagesSlicesFile,
-  );
+  const requestedSlicesFile =
+    options.depsSlicesFile || options.usagesSlicesFile;
+  let slicesFile = requestedSlicesFile
+    ? resolve(requestedSlicesFile)
+    : undefined;
   // Issue: 615 fix. Reuse existing slices file
   if (slicesFile && safeExistsSync(slicesFile)) {
     const slicesData = JSON.parse(readFileSync(slicesFile, "utf-8"));
@@ -401,8 +487,18 @@ export async function getPyModules(src, epkgList, options) {
     } else {
       modList = slicesData;
     }
-  } else {
+  } else if (hasPythonSources(src, options)) {
+    // A new slices file is kept where it was asked for only when the evidence
+    // step will read it again; otherwise atom writes it to a temporary
+    // directory instead of the working directory.
+    if (!options.evidence) {
+      slicesFile = undefined;
+    }
     modList = findAppModules(src, "python", "parsedeps", slicesFile, options);
+  } else {
+    thoughtLog(
+      `There are no Python sources at '${src}', so atom parsedeps has nothing to read.`,
+    );
   }
   const pyDefaultModules = new Set(PYTHON_STD_MODULES);
   modList = modList.filter(

package/lib/ecosystems/remote/huggingface.js

@@ -5,6 +5,7 @@ import {
   recordActivity,
 } from "../../core/activity.js";
 import { sanitizeStructuredValueForBom } from "../../core/propertySanitizer.js";
+import { registerRunReset } from "../../core/runState.js";
 import {
   detectAiModelVariants,
   normalizeDetectedVariants,
@@ -167,6 +168,8 @@ export function resetHuggingFaceRemoteCaches() {
   payloadCache.clear();
 }
 
+registerRunReset("huggingface:responses", resetHuggingFaceRemoteCaches);
+
 const normalizeLicenseFilePath = (licensePath) =>
   String(licensePath || "")
     .trim()

package/lib/ecosystems/rubyutils.js

@@ -15,23 +15,21 @@ import { build, Purl } from "@cdxgen/cdx-purl";
 
 // utils.js does not import rubyutils.js, so importing from it here is safe and
 // does not introduce a cyclic dependency.
-import {
-  cdxgenAgent,
-  DEBUG_MODE,
-  readEnvironmentVariable,
-} from "../core/activity.js";
+import { DEBUG_MODE, readEnvironmentVariable } from "../core/activity.js";
 import { shouldFetchLicense } from "../core/env.js";
 import { getAllFiles, safeExistsSync, safeSpawnSync } from "../core/fs.js";
 import { thoughtLog } from "../core/logger.js";
 import { dirNameStr, isWin } from "../core/paths.js";
 import { mergeDependencies } from "../inventory/depsUtils.js";
 import {
+  gatedGet,
   prefetchEnabled,
   prefetchedResponse,
   prefetchJson,
   recordPolicyDegradationFromError,
 } from "../inventory/fetchBatch.js";
 import { genericPurl } from "../inventory/purl.js";
+import { isValidIdnEmail } from "../parsers/email.js";
 
 // FIXME. This has to get removed, once we improve the module detection one-liner.
 // If you're a Rubyist, please help us improve this code.
@@ -913,6 +911,9 @@ export async function enrichGemsFromLocalCache(pkgList, options = {}) {
         installedPkgs = await parseGemspecData(
           readFileSync(gemspecFile, { encoding: "utf-8" }),
           gemspecFile,
+          // The gemspec is read only for the fields it holds; the registry
+          // round the caller is about to make answers the rest.
+          { noFetch: true },
         );
       } catch (_err) {
         // pass
@@ -983,16 +984,21 @@ function applyGemMetadata(p, body) {
       p.properties.push({ name, value: `${value}` });
     }
   };
-  p.description = body.description || body.summary || "";
-  if (body.licenses) {
+  // The registry fills only what the lockfile, the gemspec and the caches on
+  // this machine left open, so a local answer survives the round.
+  if (!p.description?.length && (body.description || body.summary)) {
+    p.description = body.description || body.summary;
+  }
+  if (body.licenses && !p.license) {
     p.license = body.licenses;
   }
   if (body.metadata) {
-    if (body.metadata.source_code_uri) {
+    if (body.metadata.source_code_uri && !p.repository) {
       p.repository = { url: body.metadata.source_code_uri };
       if (
         body.homepage_uri &&
-        body.homepage_uri !== body.metadata.source_code_uri
+        body.homepage_uri !== body.metadata.source_code_uri &&
+        !p.homepage
       ) {
         p.homepage = { url: body.homepage_uri };
       }
@@ -1033,7 +1039,7 @@ function applyGemMetadata(p, body) {
   if (!p.externalReferences.length) {
     p.externalReferences = undefined;
   }
-  if (body.sha) {
+  if (body.sha && !p._integrity && !p.hashes?.length) {
     p._integrity = `sha256-${body.sha}`;
   }
   // RubyGems reports the authors as a comma separated string. It is split into
@@ -1173,7 +1179,17 @@ export async function getRubyGemsMetadata(pkgList) {
     for (const [name, pkgs] of Object.entries(pkgsByName)) {
       const versionedPkgs = pkgs.filter((p) => p.version);
       if (versionedPkgs.length > 1) {
+        // The listing answers every versioned variant of this gem in one
+        // request. A direct URL is asked for only an entry the listing does
+        // not answer, so it is not prefetched here: the serial path below
+        // requests one when the listing misses.
         addBatchUrl(`${RUBYGEMS_V1_VERSIONS_URL}${name}.json`);
+        for (const p of pkgs) {
+          if (!p.version) {
+            addBatchUrl(gemDirectUrl(p, RUBYGEMS_V2_URL, RUBYGEMS_V1_URL));
+          }
+        }
+        continue;
       }
       for (const p of pkgs) {
         addBatchUrl(gemDirectUrl(p, RUBYGEMS_V2_URL, RUBYGEMS_V1_URL));
@@ -1201,7 +1217,7 @@ export async function getRubyGemsMetadata(pkgList) {
           }
           const res =
             prefetchedResponse(prefetched, versionsUrl) ||
-            (await cdxgenAgent.get(versionsUrl, apiOptions));
+            (await gatedGet(versionsUrl, apiOptions));
           versions = res.body;
         }
         if (Array.isArray(versions) && versions.length) {
@@ -1280,7 +1296,7 @@ async function enrichGemFromVersionEndpoint(
     }
     const res =
       prefetchedResponse(prefetched, fullUrl) ||
-      (await cdxgenAgent.get(fullUrl, apiOptions));
+      (await gatedGet(fullUrl, apiOptions));
     let body = res.body;
     if (body?.length) {
       body = body[0];
@@ -1565,6 +1581,12 @@ function listGemBindirFiles(gemspecFile, bindir = "bin") {
  * @param {Object} [options] Parse options
  * @param {Object<string, string>} [options.lockedVersions] Locked versions of
  *   the gems the project directory serves, by gem name
+ * @param {boolean} [options.projectGemspec] The gemspec describes the project
+ *   being scanned. No registry holds the project, so its gem is never looked
+ *   up and keeps what the gemspec says.
+ * @param {boolean} [options.noFetch] Read the gemspec for the fields it holds
+ *   without asking the registry about the gem. The local cache enricher uses
+ *   this, since the caller is responsible for the registry round.
  */
 export async function parseGemspecData(gemspecData, gemspecFile, options = {}) {
   let pkgList = [];
@@ -1946,9 +1968,11 @@ export async function parseGemspecData(gemspecData, gemspecFile, options = {}) {
     // `spec.email` holds the contact addresses of the gem, not of an author.
     // An address can only be attributed when there is one per author, in the
     // same order, or when the gem has a single author.
-    const emails = (lists.email || []).filter(
-      (email) => email.includes("@") && !email.includes(" "),
-    );
+    // Only a valid `idn-email` may reach `organizationalContact.email`; the
+    // schema asserts the format, so anything else fails BOM validation.
+    const emails = (lists.email || [])
+      .map((email) => `${email}`.trim())
+      .filter(isValidIdnEmail);
     pkg.authors = lists.authors.map((name, i) => {
       let email;
       if (emails.length === lists.authors.length) {
@@ -1983,7 +2007,7 @@ export async function parseGemspecData(gemspecData, gemspecFile, options = {}) {
   } else {
     console.log("Unable to parse", gemspecData, gemspecFile);
   }
-  if (shouldFetchLicense()) {
+  if (shouldFetchLicense() && !options.projectGemspec && !options.noFetch) {
     return await getRubyGemsMetadata(pkgList);
   }
   return pkgList;
@@ -1997,11 +2021,24 @@ export async function parseGemspecData(gemspecData, gemspecFile, options = {}) {
  * directory (`PATH` with `remote: .`), which is how Bundler records the gem a
  * project builds when its Gemfile uses the `gemspec` directive.
  *
+ * Gems the lockfile serves from a git remote or from a path outside the
+ * project are never sent to rubygems.org: no public registry holds the release
+ * they pinned. Before the registry round, the caches on this machine are read
+ * first, so a gem whose installed gemspec or compact index entry is present
+ * keeps those answers.
+ *
  * @param {object} gemLockData Gemfile.lock data
  * @param {string} lockFile Lock file
+ * @param {object} [options] Options
+ * @param {string} [options.gemHome] GEM_HOME to read installed gemspecs from
+ * @param {string} [options.compactIndexCacheDir] Bundler compact index cache
  */
-export async function parseGemfileLockData(gemLockData, lockFile) {
-  let pkgList = [];
+export async function parseGemfileLockData(
+  gemLockData,
+  lockFile,
+  options = {},
+) {
+  const pkgList = [];
   const pkgnames = {};
   const dependenciesList = [];
   const dependenciesMap = {};
@@ -2207,13 +2244,16 @@ export async function parseGemfileLockData(gemLockData, lockFile) {
           }
           // Bundler locks every gem against exactly one source, so the gem's
           // own top-level line is where its remote belongs. A later duplicate
-          // sighting of the same name would be the same release.
+          // sighting of the same name would be the same release. The section
+          // is kept as well: a gem under GIT or PATH is served from a source
+          // no public registry holds.
           if (!gemSourceInfo[name]) {
             gemSourceInfo[name] = {
               remote: lastRemote,
               revision: lastRevision,
               branch: lastBranch,
               tag: lastTag,
+              section: lastSection,
             };
           }
         }
@@ -2369,7 +2409,20 @@ export async function parseGemfileLockData(gemLockData, lockFile) {
     });
   }
   if (shouldFetchLicense()) {
-    pkgList = await getRubyGemsMetadata(pkgList);
+    // The caches on this machine answer first: the installed gemspec holds
+    // the licence the registry would be asked for, and the compact index the
+    // checksum, so the registry round below only fills what is still missing.
+    await enrichGemsFromLocalCache(pkgList, options);
+    // The gem the lockfile serves from the project's own directory is the
+    // project itself, and a gem under GIT or PATH is served from a source no
+    // public registry holds; only the rest are looked up.
+    const lookupList = pkgList.filter(
+      (p) =>
+        !projectGemRefs.includes(p["bom-ref"]) &&
+        gemSourceInfo[p.name]?.section !== "GIT" &&
+        gemSourceInfo[p.name]?.section !== "PATH",
+    );
+    await getRubyGemsMetadata(lookupList);
   }
   return { pkgList, dependenciesList, rootList, projectGemRefs };
 }

package/lib/ecosystems/rusi.js

@@ -10,6 +10,11 @@ import {
   safeSpawnSync,
 } from "../core/fs.js";
 import { dirNameStr } from "../core/paths.js";
+import {
+  reportPluginRunFailure,
+  resolvePluginSourceDir,
+} from "../core/pluginRun.js";
+import { isProjectToolingAllowed } from "../core/projectTooling.js";
 import { applyAlgorithmProperties } from "../inventory/cryptoAlgorithmFamily.js";
 import { resolvePluginBinary } from "../inventory/plugins.js";
 
@@ -817,9 +822,24 @@ export function runRusiAnalysis(src, outputFile, options = {}) {
   if (!executable || !src || !outputFile) {
     return false;
   }
+  // rusi runs cargo and rustc on the project, which follow its
+  // `.cargo/config.toml` and `rust-toolchain.toml` and can compile its build
+  // scripts, so it waits for permission to run the project's tooling.
+  if (!isProjectToolingAllowed(options)) {
+    console.log(
+      "Skipping rusi since dependency installation is disabled. Rust evidence needs cargo to run on the project.",
+    );
+    return false;
+  }
 
+  // cdxgen may run with its working directory already inside the project
+  // while the source argument is relative to a repository root; a plain
+  // resolve would join the two into a directory that does not exist. The
+  // helper checks the result and falls back to the working directory when it
+  // is itself the cargo project under analysis.
+  const analysisDir = resolvePluginSourceDir(src, ["Cargo.toml"]);
   const rusiMode = options.rusiMode === "cryptos" ? "cryptos" : "analyze";
-  const args = [rusiMode, "--dir", resolve(src), "--out", resolve(outputFile)];
+  const args = [rusiMode, "--dir", analysisDir, "--out", resolve(outputFile)];
 
   // Set callgraph and data-flow args
   args.push("--callgraph", options.rusiCallgraph || "static");
@@ -850,18 +870,21 @@ export function runRusiAnalysis(src, outputFile, options = {}) {
   }
 
   const result = safeSpawnSync(executable, args, {
-    cwd: resolve(src),
+    cwd: analysisDir,
     shell: false,
   });
 
   if (result?.status !== 0 || result?.error || !safeExistsSync(outputFile)) {
-    if (DEBUG_MODE) {
-      if (result?.stdout || result?.stderr) {
-        console.error(result.stdout, result.stderr);
-      } else {
-        console.log("Check if the rusi plugin was installed successfully.");
-      }
-    }
+    reportPluginRunFailure({
+      tool: "rusi",
+      ecosystem: "rust",
+      executable,
+      args,
+      dir: analysisDir,
+      result,
+      outputFile,
+      options,
+    });
     return false;
   }
   return true;