@cdxgen/cdxgen 13.2.0 → 13.4.0

package/lib/ecosystems/pythonutils.js

@@ -18,7 +18,7 @@ import {
   readEnvironmentVariable,
 } from "../core/activity.js";
 import { PYTHON_CMD, PYTHON_EXCLUDED_COMPONENTS } from "../core/env.js";
-import { safeExistsSync, safeSpawnSync } from "../core/fs.js";
+import { getAllFiles, safeExistsSync, safeSpawnSync } from "../core/fs.js";
 import { thoughtLog } from "../core/logger.js";
 import { isWin } from "../core/paths.js";
 import {
@@ -331,6 +331,70 @@ export function get_python_command_from_env(env) {
   return fallbackCmd;
 }
 
+/** Directories that hold installed packages, not the project's own code. */
+const INSTALLED_PACKAGE_DIRS = [
+  "**/site-packages/**",
+  "**/dist-packages/**",
+  "**/.venv/**",
+  "**/venv/**",
+  "**/.tox/**",
+  "**/.nox/**",
+  "**/__pypackages__/**",
+  "**/node_modules/**",
+];
+
+const projectModuleNamesCache = new Map();
+
+/**
+ * The top-level packages and modules a project provides, wherever they sit
+ * in its tree: each outermost package directory (one with an `__init__.py`
+ * whose parent has none), and the module files beside such packages, since
+ * their directory is a source root. Directories of installed packages
+ * (virtual environments, `site-packages`) are not the project's.
+ *
+ * @param {string} src Path to the project being analysed
+ * @returns {Set<string>} Module names
+ */
+export function projectModuleNames(src) {
+  const root = resolve(src);
+  if (projectModuleNamesCache.has(root)) {
+    return projectModuleNamesCache.get(root);
+  }
+  const names = new Set();
+  const sourceRoots = new Set();
+  const inits = getAllFiles(root, "**/__init__.py", {
+    exclude: INSTALLED_PACKAGE_DIRS,
+  });
+  for (const init of inits) {
+    let pkgDir = dirname(resolve(init));
+    while (
+      dirname(pkgDir) !== root &&
+      dirname(pkgDir).startsWith(root) &&
+      safeExistsSync(join(dirname(pkgDir), "__init__.py"))
+    ) {
+      pkgDir = dirname(pkgDir);
+    }
+    if (pkgDir === root) {
+      continue;
+    }
+    names.add(basename(pkgDir));
+    sourceRoots.add(dirname(pkgDir));
+  }
+  for (const dir of sourceRoots) {
+    try {
+      for (const f of readdirSync(dir)) {
+        if (f.endsWith(".py") && f !== "__init__.py") {
+          names.add(f.slice(0, -3));
+        }
+      }
+    } catch {
+      // an unreadable directory provides nothing
+    }
+  }
+  projectModuleNamesCache.set(root, names);
+  return names;
+}
+
 /**
  * Is this import satisfied by a module that lives in the project itself?
  *
@@ -344,7 +408,9 @@ export function get_python_command_from_env(env) {
  * filesystem whether the module is ours answers it exactly.
  *
  * Checks the project root and a `src/` layout for a module file, a package directory,
- * and a PEP 420 namespace package directory.
+ * and a PEP 420 namespace package directory, then the top-level packages found
+ * anywhere in the project and the modules beside them (tool-heavy repositories
+ * keep their Python libraries in places such as `dev_tools/pylibs/`).
  *
  * @param {string} src Path to the project being analysed
  * @param {string} name Imported top-level module name
@@ -354,6 +420,9 @@ export function isFirstPartyModule(src, name) {
   if (!src || !name || name.includes("/") || name.includes("\\")) {
     return false;
   }
+  if (projectModuleNames(src).has(name)) {
+    return true;
+  }
   for (const root of [src, join(src, "src")]) {
     if (
       safeExistsSync(join(root, `${name}.py`)) ||
@@ -378,6 +447,21 @@ export function isFirstPartyModule(src, name) {
   return false;
 }
 
+/**
+ * Whether a directory holds Python sources for atom to parse.
+ *
+ * The search skips the directories the manifest searches skip, such as
+ * node_modules and dot directories, and honours the exclude patterns, so a
+ * Python file shipped inside an npm package does not count.
+ *
+ * @param {string} src directory
+ * @param {Object} options CLI options
+ * @returns {boolean} true when the directory holds at least one Python file
+ */
+export function hasPythonSources(src, options = {}) {
+  return getAllFiles(src, "**/*.py", options).length > 0;
+}
+
 /**
  * Method to find python modules by parsing the imports and then checking with PyPI to obtain the latest version
  *
@@ -390,9 +474,11 @@ export async function getPyModules(src, epkgList, options) {
   const allImports = {};
   const dependenciesList = [];
   let modList = [];
-  const slicesFile = resolve(
-    options.depsSlicesFile || options.usagesSlicesFile,
-  );
+  const requestedSlicesFile =
+    options.depsSlicesFile || options.usagesSlicesFile;
+  let slicesFile = requestedSlicesFile
+    ? resolve(requestedSlicesFile)
+    : undefined;
   // Issue: 615 fix. Reuse existing slices file
   if (slicesFile && safeExistsSync(slicesFile)) {
     const slicesData = JSON.parse(readFileSync(slicesFile, "utf-8"));
@@ -401,8 +487,18 @@ export async function getPyModules(src, epkgList, options) {
     } else {
       modList = slicesData;
     }
-  } else {
+  } else if (hasPythonSources(src, options)) {
+    // A new slices file is kept where it was asked for only when the evidence
+    // step will read it again; otherwise atom writes it to a temporary
+    // directory instead of the working directory.
+    if (!options.evidence) {
+      slicesFile = undefined;
+    }
     modList = findAppModules(src, "python", "parsedeps", slicesFile, options);
+  } else {
+    thoughtLog(
+      `There are no Python sources at '${src}', so atom parsedeps has nothing to read.`,
+    );
   }
   const pyDefaultModules = new Set(PYTHON_STD_MODULES);
   modList = modList.filter(

package/lib/ecosystems/remote/huggingface.js

@@ -5,6 +5,7 @@ import {
   recordActivity,
 } from "../../core/activity.js";
 import { sanitizeStructuredValueForBom } from "../../core/propertySanitizer.js";
+import { registerRunReset } from "../../core/runState.js";
 import {
   detectAiModelVariants,
   normalizeDetectedVariants,
@@ -167,6 +168,8 @@ export function resetHuggingFaceRemoteCaches() {
   payloadCache.clear();
 }
 
+registerRunReset("huggingface:responses", resetHuggingFaceRemoteCaches);
+
 const normalizeLicenseFilePath = (licensePath) =>
   String(licensePath || "")
     .trim()