@cdxgen/cdxgen 13.0.1 → 13.2.0

package/lib/ecosystems/pypiClassifiers.js

@@ -0,0 +1,326 @@
+// PyPI Trove classifier interpretation and package -> module-name resolution.
+//
+// PyPI responses carry two structured signals that were previously read only for
+// their license lines and then discarded:
+//
+//   * `Framework :: X` — the author's declaration that the distribution belongs to
+//     framework X. It is more reliable than any name-prefix heuristic: it catches
+//     `flask-talisman`, `dj-database-url` and `starlette-context` without anyone
+//     maintaining a list, and it never fires on a package that merely has a
+//     framework's name embedded in its own.
+//   * `Topic :: ...` — a curated taxonomy of what the distribution does, which maps
+//     cleanly onto the description-tag vocabulary in `data/component-tags.json`.
+//
+// Tags derived here flow into `component.tags` and, for the `framework` tag, into
+// `determinePackageType()` in `lib/cli/bomAssembly.js`, which already promotes any
+// package carrying a `framework` tag to `type: "framework"`.
+
+import { PYPI_MODULE_PACKAGE_MAPPING } from "../core/state.js";
+
+/**
+ * `Framework ::` values that do not mean "an application is built on this".
+ *
+ * Two groups, both of which would otherwise put a `framework` tag — and therefore
+ * `type: "framework"` — on a large share of an ordinary dependency tree, drowning
+ * the runtime frameworks the tag exists to surface:
+ *
+ *   * Development, build, test and documentation tooling. A pytest plugin
+ *     declaring `Framework :: Pytest` is making a true statement, but it belongs
+ *     to the dev dependencies, not to what the application runs on.
+ *   * Concurrency runtimes. `Framework :: AsyncIO` is declared by a very large
+ *     number of libraries to advertise that they work under asyncio; observed on
+ *     `fastapi`, and `Framework :: AnyIO` likewise on `starlette`. It describes an
+ *     execution model, not a framework the application is written against.
+ *
+ * The declared name is still emitted as a tag in every case, and still recorded in
+ * `cdx:pypi:frameworks`; only the generic `framework` tag is withheld.
+ */
+export const PYPI_NON_APPLICATION_FRAMEWORKS = new Set([
+  "anyio",
+  "asyncio",
+  "buildout",
+  "cookiecutter",
+  "flake8",
+  "hatch",
+  "hypothesis",
+  "mkdocs",
+  "pytest",
+  "robotframework",
+  "setuptools",
+  "sphinx",
+  "tox",
+  "trio",
+  "wheel",
+]);
+
+/**
+ * `Topic ::` prefix -> tags, longest prefix wins.
+ *
+ * Every value on the right-hand side is an existing entry in
+ * `data/component-tags.json` so that a tag derived from a classifier and the same
+ * tag derived from a description are indistinguishable to consumers.
+ */
+const TOPIC_TAGS = [
+  [
+    "Topic :: Software Development :: Libraries :: Application Frameworks",
+    ["framework"],
+  ],
+  ["Topic :: Software Development :: Object Brokering", ["rpc"]],
+  ["Topic :: Software Development :: Testing", ["test"]],
+  ["Topic :: Internet :: WWW/HTTP :: WSGI", ["web", "http", "wsgi"]],
+  ["Topic :: Internet :: WWW/HTTP :: Session", ["web", "http", "auth"]],
+  ["Topic :: Internet :: WWW/HTTP :: Dynamic Content", ["web", "http"]],
+  ["Topic :: Internet :: WWW/HTTP", ["web", "http"]],
+  ["Topic :: Internet :: XML-RPC", ["rpc", "xml"]],
+  ["Topic :: Internet :: Proxy Servers", ["web", "traffic"]],
+  ["Topic :: Database :: Front-Ends", ["database"]],
+  ["Topic :: Database", ["database"]],
+  ["Topic :: Security :: Cryptography", ["crypto", "security"]],
+  ["Topic :: Security", ["security"]],
+  ["Topic :: System :: Logging", ["logging"]],
+  ["Topic :: System :: Distributed Computing", ["rpc"]],
+  ["Topic :: Communications :: Email", ["mail"]],
+  ["Topic :: Text Processing :: Markup :: HTML", ["html", "template"]],
+  ["Topic :: Text Processing :: Markup :: XML", ["xml"]],
+  ["Topic :: Scientific/Engineering :: Artificial Intelligence", ["ai", "ml"]],
+  [
+    "Topic :: Scientific/Engineering :: Image Recognition",
+    ["ml", "object-detect"],
+  ],
+  ["Topic :: Office/Business :: Financial", ["finance"]],
+  ["Topic :: Terminals", ["cli"]],
+  ["Topic :: Utilities", ["tools"]],
+];
+
+/**
+ * Normalize a distribution name per PEP 503.
+ *
+ * @param {string} name Distribution name
+ * @returns {string} Normalized name, or an empty string for unusable input
+ */
+export function normalizePypiName(name) {
+  if (!name || typeof name !== "string") {
+    return "";
+  }
+  let out = "";
+  let lastWasSeparator = false;
+  for (const ch of name.trim().toLowerCase()) {
+    if (ch === "-" || ch === "_" || ch === ".") {
+      if (!lastWasSeparator && out.length) {
+        out += "-";
+      }
+      lastWasSeparator = true;
+      continue;
+    }
+    out += ch;
+    lastWasSeparator = false;
+  }
+  return out.endsWith("-") ? out.slice(0, -1) : out;
+}
+
+/**
+ * `data/pypi-pkg-aliases.json` maps module name -> distribution name. Inverting it
+ * gives the mapping consumers actually need: distribution -> the module names it
+ * installs. Built once on first use.
+ *
+ * @type {Map<string, string[]> | undefined}
+ */
+let invertedAliases;
+
+function packageToModules() {
+  if (invertedAliases) {
+    return invertedAliases;
+  }
+  invertedAliases = new Map();
+  for (const [moduleName, pkgName] of Object.entries(
+    PYPI_MODULE_PACKAGE_MAPPING,
+  )) {
+    const key = normalizePypiName(pkgName);
+    if (!key || !moduleName) {
+      continue;
+    }
+    const existing = invertedAliases.get(key);
+    if (existing) {
+      if (!existing.includes(moduleName)) {
+        existing.push(moduleName);
+      }
+    } else {
+      invertedAliases.set(key, [moduleName]);
+    }
+  }
+  return invertedAliases;
+}
+
+/**
+ * Best-known import module names for a PyPI distribution.
+ *
+ * Two sources, in order: the curated alias map (which is where the cases that
+ * cannot be derived from the name live — `pyyaml` installs `yaml`, `absl-py`
+ * installs `absl`, `beautifulsoup4` installs `bs4`), and the name-derived form,
+ * which is correct for the large majority of distributions. Callers that need to
+ * distinguish the two should read the alias map themselves; for tagging purposes
+ * both are usable and a wrong extra candidate simply matches nothing.
+ *
+ * This is deliberately not a guess at *every* module a distribution ships — only
+ * at its top-level import names.
+ *
+ * @param {string} name Distribution name, e.g. "PyYAML"
+ * @returns {string[]} Candidate top-level module names, deduplicated
+ */
+export function pypiModulesForPackage(name) {
+  const normalized = normalizePypiName(name);
+  if (!normalized) {
+    return [];
+  }
+  const modules = [...(packageToModules().get(normalized) || [])];
+  // The name-derived form. PEP 503 collapses `.` and `_` to `-`; imports use `_`.
+  const derived = normalized.replaceAll("-", "_");
+  if (derived && !modules.includes(derived)) {
+    modules.push(derived);
+  }
+  return modules;
+}
+
+/**
+ * Interpret a PyPI classifier list.
+ *
+ * @param {string[]} classifiers Raw `info.classifiers` from the PyPI JSON API
+ * @returns {{frameworks: string[], tags: string[]}} Declared frameworks (lowercased,
+ *   e.g. `["django"]`) and the tag vocabulary they and the `Topic ::` lines imply
+ */
+export function parsePypiClassifiers(classifiers) {
+  const frameworks = [];
+  const tags = new Set();
+  if (!Array.isArray(classifiers)) {
+    return { frameworks, tags: [] };
+  }
+  for (const classifier of classifiers) {
+    if (!classifier || typeof classifier !== "string") {
+      continue;
+    }
+    if (classifier.startsWith("Framework :: ")) {
+      // "Framework :: Django :: 5.0" -> "django". The version segment is dropped:
+      // it is the framework version the package supports, not a distinct signal.
+      const declared = classifier.split(" :: ")[1]?.trim();
+      if (!declared) {
+        continue;
+      }
+      const name = declared.toLowerCase().replaceAll(" ", "-");
+      if (!frameworks.includes(name)) {
+        frameworks.push(name);
+      }
+      tags.add(name);
+      if (!PYPI_NON_APPLICATION_FRAMEWORKS.has(name)) {
+        tags.add("framework");
+      }
+      continue;
+    }
+    if (!classifier.startsWith("Topic :: ")) {
+      continue;
+    }
+    for (const [prefix, topicTags] of TOPIC_TAGS) {
+      if (classifier === prefix || classifier.startsWith(`${prefix} ::`)) {
+        for (const tag of topicTags) {
+          tags.add(tag);
+        }
+        break;
+      }
+    }
+  }
+  return { frameworks, tags: Array.from(tags) };
+}
+
+function addProperty(pkg, name, value) {
+  if (!value) {
+    return;
+  }
+  if (!pkg.properties) {
+    pkg.properties = [];
+  }
+  const existing = pkg.properties.find((prop) => prop.name === name);
+  if (existing) {
+    existing.value = value;
+    return;
+  }
+  pkg.properties.push({ name, value });
+}
+
+/**
+ * Attach classifier-derived tags and properties to a package.
+ *
+ * Tags are merged into any the package already carries rather than replacing
+ * them — `lib/managers/binary.js` sets `tags: ["source"]` as an identity marker,
+ * and losing it would change how that package is reconciled later.
+ *
+ * @param {Object} pkg Package object, mutated in place
+ * @param {string[]} classifiers Raw `info.classifiers` from the PyPI JSON API
+ * @returns {Object} The same package object
+ */
+export function applyPypiClassifierMetadata(pkg, classifiers) {
+  if (!pkg || !Array.isArray(classifiers) || !classifiers.length) {
+    return pkg;
+  }
+  const { frameworks, tags } = parsePypiClassifiers(classifiers);
+  if (tags.length) {
+    const merged = new Set(Array.isArray(pkg.tags) ? pkg.tags : []);
+    for (const tag of tags) {
+      merged.add(tag);
+    }
+    pkg.tags = Array.from(merged);
+  }
+  if (frameworks.length) {
+    addProperty(pkg, "cdx:pypi:frameworks", frameworks.join(","));
+  }
+  // The full list is retained so that consumers wanting the whole taxonomy are
+  // not limited to the tag vocabulary mapped above. Classifiers come from a
+  // closed PyPI-maintained set, so there is no free-text or secret exposure.
+  addProperty(
+    pkg,
+    "cdx:pypi:classifiers",
+    classifiers.filter((c) => typeof c === "string" && c.length).join("\n"),
+  );
+  return pkg;
+}
+
+/**
+ * Attach the distribution's top-level import module names as `internal:Namespaces`.
+ *
+ * This is the same property jar, gem, nuget and composer components already carry,
+ * so downstream consumers that map namespaces back to components need no
+ * pypi-specific path. Unlike the jar case the value is name-derived and
+ * alias-curated rather than read out of the artifact, which is why
+ * `cdx:pypi:modulesFrom` records the provenance.
+ *
+ * @param {Object} pkg Package object, mutated in place
+ * @returns {Object} The same package object
+ */
+export function applyPypiModuleNames(pkg) {
+  if (!pkg?.name) {
+    return pkg;
+  }
+  // Extras (`requests[security]`) are not part of the distribution name.
+  const name = pkg.name.includes("[") ? pkg.name.split("[")[0] : pkg.name;
+  if (name.includes("https")) {
+    return pkg;
+  }
+  const normalized = normalizePypiName(name);
+  const modules = pypiModulesForPackage(name);
+  if (!modules.length) {
+    return pkg;
+  }
+  // Most alias-map entries map a distribution to the module its own name already
+  // implies, so the map having an entry says nothing. The provenance worth
+  // recording is whether it contributed a module the name could not have given -
+  // that is the difference between `flask` and `pyyaml` -> `yaml`.
+  const derived = normalized.replaceAll("-", "_");
+  const curatedAddedSomething = (packageToModules().get(normalized) || []).some(
+    (moduleName) => moduleName !== derived,
+  );
+  addProperty(pkg, "internal:Namespaces", modules.join("\n"));
+  addProperty(
+    pkg,
+    "cdx:pypi:modulesFrom",
+    curatedAddedSomething ? "alias-map,distribution-name" : "distribution-name",
+  );
+  return pkg;
+}

package/lib/ecosystems/pythonutils.js

@@ -331,6 +331,53 @@ export function get_python_command_from_env(env) {
   return fallbackCmd;
 }
 
+/**
+ * Is this import satisfied by a module that lives in the project itself?
+ *
+ * `atom parsedeps` reports every non-stdlib import it sees, including imports of the
+ * project's own packages. Attributing those to a distribution invents dependencies:
+ * `data/pypi-pkg-aliases.json` maps the module `app` to `zope2`, `test` to `pytabix`,
+ * `models` to `asposestorage` and `src` to `auto-mix-prep`, so any project with a
+ * local package of that name acquires an obscure component it never depended on -
+ * along with every namespace and tag that component carries downstream. Filtering by
+ * name would need a denylist of every plausible first-party name; asking the
+ * filesystem whether the module is ours answers it exactly.
+ *
+ * Checks the project root and a `src/` layout for a module file, a package directory,
+ * and a PEP 420 namespace package directory.
+ *
+ * @param {string} src Path to the project being analysed
+ * @param {string} name Imported top-level module name
+ * @returns {boolean} `true` if the project supplies this module itself
+ */
+export function isFirstPartyModule(src, name) {
+  if (!src || !name || name.includes("/") || name.includes("\\")) {
+    return false;
+  }
+  for (const root of [src, join(src, "src")]) {
+    if (
+      safeExistsSync(join(root, `${name}.py`)) ||
+      safeExistsSync(join(root, name, "__init__.py")) ||
+      safeExistsSync(join(root, name, "__main__.py"))
+    ) {
+      return true;
+    }
+    // PEP 420 namespace package: a directory of the right name holding python
+    // sources but no __init__.py.
+    const candidate = join(root, name);
+    if (safeExistsSync(candidate)) {
+      try {
+        if (readdirSync(candidate).some((f) => f.endsWith(".py"))) {
+          return true;
+        }
+      } catch {
+        // not a readable directory - treat as not first-party
+      }
+    }
+  }
+  return false;
+}
+
 /**
  * Method to find python modules by parsing the imports and then checking with PyPI to obtain the latest version
  *
@@ -362,7 +409,8 @@ export async function getPyModules(src, epkgList, options) {
     (x) =>
       !pyDefaultModules.has(x.name.toLowerCase()) &&
       !x.name.startsWith("_") &&
-      !x.name.startsWith("."),
+      !x.name.startsWith(".") &&
+      !isFirstPartyModule(src, x.name),
   );
   let pkgList = modList.map((p) => {
     const apkg = {

package/lib/ecosystems/rubyutils.js

@@ -19,6 +19,7 @@ import {
   prefetchEnabled,
   prefetchedResponse,
   prefetchJson,
+  recordPolicyDegradationFromError,
 } from "../inventory/fetchBatch.js";
 
 // FIXME. This has to get removed, once we improve the module detection one-liner.
@@ -780,6 +781,7 @@ export async function getRubyGemsMetadata(pkgList) {
           continue;
         }
       } catch (err) {
+        recordPolicyDegradationFromError(err);
         if (DEBUG_MODE) {
           console.error(name, err);
         }
@@ -832,6 +834,7 @@ async function enrichGemFromVersionEndpoint(
     }
     applyGemMetadata(p, body);
   } catch (err) {
+    recordPolicyDegradationFromError(err);
     if (DEBUG_MODE) {
       console.error(p, err);
     }
@@ -1241,6 +1244,16 @@ export async function parseGemspecData(gemspecData, gemspecFile) {
       pkg.externalReferences.push({ type: refType, url: value, comment: key });
     }
   }
+  // `s.homepage = "..."` is the project website. Keep it as an external
+  // reference; the transient `homepage` key is a plain string that never
+  // reaches the BOM because component conversion expects the `{ url }` shape.
+  if (pkg.homepage) {
+    pkg.externalReferences = pkg.externalReferences || [];
+    if (!pkg.externalReferences.some((ref) => ref.url === pkg.homepage)) {
+      pkg.externalReferences.push({ type: "website", url: pkg.homepage });
+    }
+    delete pkg.homepage;
+  }
   if (pkg.name) {
     const purlString = toGemPurl(pkg.name, pkg.version, gemPlatform);
     pkg.purl = purlString;

package/lib/ecosystems/rusi.js

@@ -9,11 +9,16 @@ import {
   safeRmSync,
   safeSpawnSync,
 } from "../core/fs.js";
+import { dirNameStr } from "../core/paths.js";
 import { applyAlgorithmProperties } from "../inventory/cryptoAlgorithmFamily.js";
 import { resolvePluginBinary } from "../inventory/plugins.js";
 
 const RUST_LANGUAGES = new Set(["rust", "rs", "rust-lang"]);
 
+const RUSI_CRYPTO_OIDS = JSON.parse(
+  readFileSync(join(dirNameStr, "data", "crypto-oid.json"), "utf-8"),
+);
+
 /**
  * Resolves the rusi binary.
  *
@@ -82,6 +87,75 @@ function createPurlAliasMap(components = []) {
   return purlAliasMap;
 }
 
+/**
+ * Path roots that never name a dependency crate.
+ */
+const RUST_NON_CRATE_PATH_ROOTS = new Set([
+  "crate",
+  "self",
+  "super",
+  "std",
+  "core",
+  "alloc",
+  "proc_macro",
+  "test",
+]);
+
+/**
+ * Index the cargo components by the identifier a Rust path spells them with.
+ *
+ * A crate named `cfg-if` is written `cfg_if` in source, and a lock file can
+ * carry several versions of the same crate, so a name maps to every matching
+ * purl.
+ *
+ * @param {Object[]} components Components from the BOM
+ * @returns {Map<string, string[]>} Crate identifier to purls
+ */
+function createCrateNameIndex(components = []) {
+  const crateNameIndex = new Map();
+  for (const component of components) {
+    if (!component?.purl?.startsWith("pkg:cargo/") || !component.name) {
+      continue;
+    }
+    const crateIdentifier = component.name.toLowerCase().replaceAll("-", "_");
+    const purls = crateNameIndex.get(crateIdentifier) || [];
+    if (!purls.includes(component.purl)) {
+      purls.push(component.purl);
+    }
+    crateNameIndex.set(crateIdentifier, purls);
+  }
+  return crateNameIndex;
+}
+
+/**
+ * Resolve the dependency crates a Rust path refers to.
+ *
+ * rusi reports an import against the package that declares it, so the imported
+ * crate has to be recovered from the path itself. This is what turns an
+ * `use hex::encode;` into usage evidence for the `hex` component.
+ *
+ * @param {string} importPath Rust path, such as `hex::encode`
+ * @param {Map<string, string[]>} crateNameIndex Index from {@link createCrateNameIndex}
+ * @param {string} [ownerPurl] purl of the package that declares the path
+ * @returns {string[]} Matching component purls
+ */
+function resolveImportedCratePurls(importPath, crateNameIndex, ownerPurl) {
+  if (!importPath || !crateNameIndex?.size) {
+    return [];
+  }
+  const pathRoot = String(importPath)
+    .replace(/^::/, "")
+    .split("::")[0]
+    .trim()
+    .toLowerCase();
+  if (!pathRoot || RUST_NON_CRATE_PATH_ROOTS.has(pathRoot)) {
+    return [];
+  }
+  return (crateNameIndex.get(pathRoot) || []).filter(
+    (purl) => purl !== ownerPurl,
+  );
+}
+
 function resolveComponentPurl(purl, purlAliasMap) {
   if (!purl) {
     return undefined;
@@ -182,19 +256,37 @@ function mergeCryptoComponent(componentsByRef, component, item) {
   return existing;
 }
 
+// The schema requires an oid for algorithm assets, so an algorithm the OID
+// table cannot name is skipped rather than emitted invalid. rusi reports
+// display-cased names ("SHA-256") while the table is lowercase ("sha-256").
+function resolveCryptoAlgorithmMetadata(name) {
+  if (!name) {
+    return undefined;
+  }
+  const lowered = String(name).toLowerCase();
+  return (
+    RUSI_CRYPTO_OIDS[name] ||
+    RUSI_CRYPTO_OIDS[lowered] ||
+    RUSI_CRYPTO_OIDS[lowered.replace(/[\s_]+/g, "-")]
+  );
+}
+
 function cryptoAlgorithmComponent(asset) {
+  const algorithmName = asset.algorithm || asset.kind || "unknown";
+  const algorithmMetadata = resolveCryptoAlgorithmMetadata(algorithmName);
+  if (!algorithmMetadata?.oid) {
+    return undefined;
+  }
   const component = {
     type: "cryptographic-asset",
-    name: asset.algorithm || asset.kind || "unknown",
-    "bom-ref": cryptoBomRef(
-      "algorithm",
-      asset.algorithm || asset.kind,
-      asset.provider || "unknown",
-    ),
+    name: algorithmName,
+    "bom-ref": cryptoBomRef("algorithm", algorithmName, algorithmMetadata.oid),
     description:
+      algorithmMetadata.description ||
       "Cryptographic algorithm/component detected by rusi source analysis",
     cryptoProperties: {
       assetType: "algorithm",
+      oid: algorithmMetadata.oid,
       algorithmProperties: { primitive: asset.kind || "unknown" },
     },
     properties: [],
@@ -361,6 +453,7 @@ function addImportEvidence(
   purlAliasMap,
   purlLocationMap,
   componentPropertiesMap,
+  crateNameIndex,
 ) {
   for (const importUsage of rusiReport.imports || []) {
     const purl = resolveComponentPurl(importUsage.purl, purlAliasMap);
@@ -380,6 +473,31 @@ function addImportEvidence(
       "cdx:rusi:importAlias",
       importUsage.alias,
     );
+    // The import is also evidence about the crate being imported, which is the
+    // component a consumer filters on.
+    for (const cratePurl of resolveImportedCratePurls(
+      importUsage.path,
+      crateNameIndex,
+      purl,
+    )) {
+      addSetValue(
+        purlLocationMap,
+        cratePurl,
+        positionLocation(importUsage.position),
+      );
+      addPropertyValue(
+        componentPropertiesMap,
+        cratePurl,
+        "cdx:rusi:importedBy",
+        purl,
+      );
+      addPropertyValue(
+        componentPropertiesMap,
+        cratePurl,
+        "cdx:rusi:importPath",
+        importUsage.path,
+      );
+    }
   }
 }
 
@@ -761,6 +879,7 @@ export function analyzeRusiProject(src, options = {}) {
  */
 export function collectRusiEvidence(rusiReport = {}, components = []) {
   const purlAliasMap = createPurlAliasMap(components);
+  const crateNameIndex = createCrateNameIndex(components);
   const purlLocationMap = {};
   const dataFlowFrames = {};
   const componentPropertiesMap = {};
@@ -774,6 +893,7 @@ export function collectRusiEvidence(rusiReport = {}, components = []) {
     purlAliasMap,
     purlLocationMap,
     componentPropertiesMap,
+    crateNameIndex,
   );
   addUsageEvidence(
     rusiReport,