@cdxgen/cdxgen 13.6.0 → 13.7.0

package/lib/inventory/deps.js

@@ -13,9 +13,11 @@ import {
   getTmpDir,
   multiChecksumFile,
   safeExistsSync,
+  safeMkdirSync,
   safeMkdtempSync,
   safeRmSync,
   safeSpawnSync,
+  safeWriteSync,
 } from "../core/fs.js";
 import { isWin } from "../core/paths.js";
 import { xml2js } from "../parsers/xml.js";
@@ -34,6 +36,10 @@ import { findLicenseId, spdxLicenses } from "./spdx.js";
 
 const jarNSMapping_cache = new Map();
 
+// The largest classes.jar on Google's Maven repository is a few tens of MiB,
+// so this bound only stops an archive that claims an implausible size.
+const MAX_AAR_INNER_JAR_BYTES = 256 * 1024 * 1024;
+
 /**
  * Whether a jar holds sources or javadoc rather than classes.
  *
@@ -193,9 +199,10 @@ export async function collectMvnDependencies(
 }
 
 /**
- * Method to collect class names from all jars in a directory
+ * Method to collect class names from all jars and Android libraries in a
+ * directory
  *
- * @param {string} jarPath Path containing jars
+ * @param {string} jarPath Path containing jars, or a single jar or aar file
  * @param {object} pomPathMap Map containing jar to pom names. Required to successfully parse gradle cache.
  *
  * @return object containing jar name and class list
@@ -213,20 +220,21 @@ export async function collectJarNS(jarPath, pomPathMap = {}) {
     )}`;
   }
   // Parse jar files to get class names
-  const jarFiles = jarPath.endsWith(".jar")
+  const jarFiles = /\.(jar|aar)$/i.test(jarPath)
     ? [jarPath]
-    : getAllFiles(jarPath, "**/*.jar");
+    : getAllFiles(jarPath, "**/*.{jar,aar}");
   if (jarFiles?.length) {
     for (const jf of jarFiles) {
       const jarName = basename(jf);
+      const isAar = jarName.toLowerCase().endsWith(".aar");
       // Sources and javadoc jars hold no runtime classes. Reading them would
       // attach their hashes to the main artifact's purl.
       if (isDocumentationJar(jarName)) {
         continue;
       }
-      let pomname =
-        pomPathMap[jarName.replace(".jar", ".pom")] ||
-        jf.replace(".jar", ".pom");
+      // Only the extension is replaced, never a match earlier in the path.
+      const toPom = (file) => `${file.slice(0, -".jar".length)}.pom`;
+      let pomname = pomPathMap[toPom(jarName)] || toPom(jf);
       let pomData;
       let purl;
       let manifestLicenses;
@@ -235,7 +243,12 @@ export async function collectJarNS(jarPath, pomPathMap = {}) {
         let searchDir = dirname(jf);
         // in case of gradle, there would be hash directory that is different for jar vs pom
         // so we need to start search from a level up
-        if (searchDir.includes(join(".gradle", "caches"))) {
+        if (
+          searchDir.includes(join(".gradle", "caches")) ||
+          // A copied cache, or one reached through GRADLE_CACHE_DIR, keeps
+          // the files-2.1 directory without the .gradle parent.
+          searchDir.split(/[\\/]/).includes("files-2.1")
+        ) {
           searchDir = join(searchDir, "..");
         }
         const pomSearch = getAllFiles(searchDir, "**/*.pom");
@@ -262,7 +275,9 @@ export async function collectJarNS(jarPath, pomPathMap = {}) {
         }
       } else {
         const coordinates = inferMavenCoordinatesFromPath(jf);
-        if (coordinates?.extension === "jar") {
+        // An aar gets the same type=jar purl the Gradle dependency tree gives
+        // it, so the two entries merge into one component.
+        if (coordinates && ["jar", "aar"].includes(coordinates.extension)) {
           purl = jarPurl(
             coordinates.group,
             coordinates.name,
@@ -274,8 +289,9 @@ export async function collectJarNS(jarPath, pomPathMap = {}) {
         // descriptor, so its metadata and manifest licence are read here,
         // before any remote lookup. Only a jar whose path names its
         // coordinates is read: they choose its descriptor, and an entry
-        // without a purl never becomes a component.
-        if (purl) {
+        // without a purl never becomes a component. An aar keeps no
+        // descriptor of its own; its POM sits beside it in the cache.
+        if (purl && !isAar) {
           const embedded = await readEmbeddedJarMetadata(jf, coordinates);
           if (embedded?.pomXml) {
             try {
@@ -300,7 +316,9 @@ export async function collectJarNS(jarPath, pomPathMap = {}) {
           console.log(`Parsing ${jf}`);
         }
         const [nsList, hashValues] = await Promise.all([
-          getJarClasses(jf),
+          // An aar is hashed as published, over the aar file itself; its
+          // classes come from the jars it holds.
+          isAar ? getAarClasses(jf) : getJarClasses(jf),
           multiChecksumFile(["md5", "sha1", "sha256", "sha512"], jf).catch(
             () => undefined,
           ),
@@ -336,7 +354,7 @@ export async function collectJarNS(jarPath, pomPathMap = {}) {
     }
   } else {
     console.log(
-      `${jarPath} did not contain any jars. Try building the project to improve the BOM precision.`,
+      `${jarPath} did not contain any jars or aar files. Try building the project to improve the BOM precision.`,
     );
   }
   return jarNSMapping;
@@ -1136,6 +1154,81 @@ export async function getJarClasses(jarFile) {
   return retList;
 }
 
+/**
+ * Method to get the classes of an Android library. An `.aar` keeps its code
+ * in `classes.jar` at its root and any number of further jars under `libs/`,
+ * each of which is read in turn.
+ *
+ * Each inner jar is unpacked to a temporary file, so in dry run mode, which
+ * blocks those writes, an aar yields no classes. An inner jar that declares
+ * more than `maxInnerJarBytes` uncompressed is skipped without being
+ * decompressed, because the whole entry is held in memory while it is copied.
+ *
+ * @param {string} aarFile Aar file to read
+ * @param {Object} [opts] Options
+ * @param {number} [opts.maxInnerJarBytes] Uncompressed size bound for each
+ *   inner jar. Defaults to 256 MiB.
+ *
+ * @returns {Promise<string[]>} Sorted class names of every jar inside
+ */
+export async function getAarClasses(
+  aarFile,
+  { maxInnerJarBytes = MAX_AAR_INNER_JAR_BYTES } = {},
+) {
+  const classes = new Set();
+  let zip;
+  let tempDir;
+  try {
+    zip = new StreamZip.async({ file: aarFile });
+    const innerJars = Object.values(await zip.entries())
+      .filter(
+        (entry) =>
+          !entry.isDirectory &&
+          (entry.name === "classes.jar" ||
+            (entry.name.startsWith("libs/") && entry.name.endsWith(".jar"))),
+      )
+      .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    for (const entry of innerJars) {
+      if (entry.size > maxInnerJarBytes) {
+        if (DEBUG_MODE) {
+          console.log(
+            `Skipping ${entry.name} in ${aarFile}: ${entry.size} bytes exceeds the ${maxInnerJarBytes} byte limit.`,
+          );
+        }
+        continue;
+      }
+      tempDir ||= safeMkdtempSync(join(getTmpDir(), "aar-classes-"));
+      // Entry names use forward slashes on every platform. node-stream-zip
+      // rejects names with "..", a backslash or an absolute prefix, so the
+      // file stays inside tempDir.
+      const innerJar = join(tempDir, ...entry.name.split("/"));
+      safeMkdirSync(dirname(innerJar), { recursive: true });
+      safeWriteSync(innerJar, await zip.entryData(entry.name));
+      // The write is skipped in dry run mode or without write permission.
+      if (!safeExistsSync(innerJar)) {
+        continue;
+      }
+      for (const cls of await getJarClasses(innerJar)) {
+        classes.add(cls);
+      }
+    }
+  } catch (err) {
+    if (DEBUG_MODE) {
+      console.log(`Unable to read the jars inside ${aarFile}: ${err.message}`);
+    }
+  } finally {
+    if (tempDir) {
+      safeRmSync(tempDir, { recursive: true, force: true });
+    }
+    try {
+      await zip?.close();
+    } catch (_err) {
+      // An archive that failed to open reports its close as well.
+    }
+  }
+  return [...classes].sort();
+}
+
 /**
  * Recursively flatten a Python package's transitive dependencies into the
  * dependencies map and flat package list.

package/lib/inventory/dosai.js

@@ -2,12 +2,21 @@ import { Buffer } from "node:buffer";
 import {
   closeSync,
   openSync,
+  readdirSync,
   readFileSync,
   readSync,
   realpathSync,
   statSync,
 } from "node:fs";
-import { basename, delimiter, isAbsolute, join, resolve } from "node:path";
+import {
+  basename,
+  delimiter,
+  extname,
+  isAbsolute,
+  join,
+  relative,
+  resolve,
+} from "node:path";
 import process from "node:process";
 
 import { DEBUG_MODE, readEnvironmentVariable } from "../core/activity.js";
@@ -24,12 +33,22 @@ import {
 } from "../core/fs.js";
 import { formatDuration, reapProcessTree } from "../core/processTree.js";
 import { encodeUriReference } from "../core/propertySanitizer.js";
+import {
+  normalizeHuggingFaceReference,
+  toHuggingFaceSupplier,
+} from "../parsers/huggingfaceUtils.js";
 import {
   indexJsonObject,
   MAX_JSON_TEXT_BYTES,
   readJsonMember,
   readJsonRange,
 } from "../parsers/largeJson.js";
+import {
+  modelCreatorEntity,
+  modelCreatorFromOrganization,
+  resolveModelCreator,
+} from "./aiProviders.js";
+import { isPathExcludedByGlobPatterns } from "./atomUtils.js";
 import {
   addDosaiSetValue,
   buildDosaiPurlAliasMap,
@@ -712,6 +731,198 @@ export function dosaiSectionsCdxgenReads(omittedSections) {
   );
 }
 
+// The files dosai reads from a source tree whose changes make an existing
+// methods slice stale: the C#, VB, and F# sources it analyses, the project
+// and MSBuild files that shape the analysis, and the Razor and WebForms
+// templates whose call sites it maps from build output. packages.config is
+// read by cdxgen's own dotnet analysis and holds the same package list a
+// csproj does, so a change to it can change the slice too. Solution files are
+// left out: dosai opens them only to budget its restore runs, and a changed
+// solution does not change the occurrences a slice holds.
+const DOSAI_SLICE_SOURCE_EXTENSIONS = new Set([
+  ".cs",
+  ".cshtml",
+  ".csproj",
+  ".fs",
+  ".fsi",
+  ".fsproj",
+  ".fsx",
+  ".props",
+  ".razor",
+  ".targets",
+  ".vb",
+  ".vbproj",
+]);
+const DOSAI_SLICE_SOURCE_FILES = new Set(["packages.config"]);
+
+// Directories that hold nothing dosai analyses. Its own tree walk skips the
+// build-output directories (obj for every file, bin for sources) and the
+// package and version-control directories, so a build cannot make a slice
+// stale and a dependency tree cannot slow the check down.
+const DOSAI_SLICE_SOURCE_SKIP_DIRS = new Set([
+  ".git",
+  "bin",
+  "node_modules",
+  "obj",
+]);
+
+// The tree walk is bounded so a pathological tree (a symlink farm, or entries
+// that appear faster than the walk reads them) cannot keep a scan from
+// finishing. A walk that reaches a bound leaves freshness unknown, and the
+// callers regenerate the slice, which is the run a fresh scan pays anyway.
+const DOSAI_FRESHNESS_MAX_DIRS = 50_000;
+const DOSAI_FRESHNESS_MAX_FILES = 200_000;
+
+/**
+ * The newest mtime among the source files dosai analyses under a directory.
+ *
+ * The walk is deterministic and bounded: directory entries are read and then
+ * visited depth-first in sorted name order, symlinked directories are not
+ * followed, and counts of visited directories and analysed files cap the
+ * work. Excluded paths are pruned rather than filtered, so nothing beneath an
+ * excluded directory is read.
+ *
+ * @param {string} projectDir Absolute directory dosai analyses
+ * @param {string[]} excludePatterns Exclude globs relative to projectDir
+ * @returns {{mtimeMs: number|undefined, complete: boolean}} The newest mtime seen (undefined when no analysed file was found), and whether the walk finished, which is false for an unreadable directory or file, or a tree past the bounds
+ */
+function newestDosaiSourceMtimeSync(projectDir, excludePatterns) {
+  const newest = { mtimeMs: undefined, complete: true };
+  const stack = [[projectDir, ""]];
+  let dirsVisited = 0;
+  let filesVisited = 0;
+  while (newest.complete && stack.length) {
+    if (dirsVisited++ >= DOSAI_FRESHNESS_MAX_DIRS) {
+      newest.complete = false;
+      break;
+    }
+    const [dir, relDir] = stack.pop();
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch (_err) {
+      newest.complete = false;
+      break;
+    }
+    entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    for (const entry of entries) {
+      const relPath = relDir ? `${relDir}/${entry.name}` : entry.name;
+      if (entry.isDirectory()) {
+        if (
+          !DOSAI_SLICE_SOURCE_SKIP_DIRS.has(entry.name.toLowerCase()) &&
+          !(
+            excludePatterns.length &&
+            isPathExcludedByGlobPatterns(relPath, excludePatterns)
+          )
+        ) {
+          stack.push([join(dir, entry.name), relPath]);
+        }
+        continue;
+      }
+      if (
+        !DOSAI_SLICE_SOURCE_EXTENSIONS.has(extname(entry.name).toLowerCase()) &&
+        !DOSAI_SLICE_SOURCE_FILES.has(entry.name.toLowerCase())
+      ) {
+        continue;
+      }
+      if (filesVisited++ >= DOSAI_FRESHNESS_MAX_FILES) {
+        newest.complete = false;
+        break;
+      }
+      try {
+        // statSync follows a symlink to the source it points at, which is
+        // the file dosai reads.
+        const mtimeMs = statSync(join(dir, entry.name)).mtimeMs;
+        if (newest.mtimeMs === undefined || mtimeMs > newest.mtimeMs) {
+          newest.mtimeMs = mtimeMs;
+        }
+      } catch (_err) {
+        newest.complete = false;
+        break;
+      }
+    }
+  }
+  return newest;
+}
+
+/**
+ * The cdxgen exclude patterns that apply under a directory, as globs relative
+ * to it.
+ *
+ * The patterns are matched the way the atom evidence filter matches
+ * `options.exclude`: a relative pattern matches at any depth, and an absolute
+ * pattern is anchored to the scanned directory, which for the walk means the
+ * patterns that sit beneath it are rebased to relative ones. Patterns that
+ * fall outside the directory are dropped.
+ *
+ * @param {string[]} excludes cdxgen exclude globs
+ * @param {string} projectDir Absolute directory dosai analyses
+ * @returns {string[]} Exclude globs relative to projectDir
+ */
+function dosaiSourceExcludePatterns(excludes, projectDir) {
+  const rebased = [];
+  if (!Array.isArray(excludes)) {
+    return rebased;
+  }
+  for (const pattern of excludes) {
+    if (typeof pattern !== "string" || !pattern.trim()) {
+      continue;
+    }
+    // resolve() leaves a relative pattern where it was and turns an absolute
+    // one into a path the relative() call drops the scanned directory from.
+    const underRoot = resolve(projectDir, pattern);
+    const rel = relative(projectDir, underRoot);
+    if (rel && !rel.startsWith("..") && !isAbsolute(rel)) {
+      rebased.push(rel.replaceAll("\\", "/"));
+    }
+  }
+  return rebased;
+}
+
+/**
+ * Whether an existing dosai methods slice is newer than the sources it
+ * describes.
+ *
+ * A slice written by an earlier scan holds the occurrences of the sources as
+ * they were then, so a slice that predates an edit must be regenerated rather
+ * than reused: reusing one keeps occurrences of call sites that no longer
+ * exist (issue 4675). The slice is fresh only when its mtime is not older
+ * than the newest mtime among the files dosai analyses under the directory.
+ * A missing slice, a slice whose mtime cannot be read, a source that cannot
+ * be stat-ed, and a tree that exceeds the walk bounds all count as stale: the
+ * callers regenerate the slice, which is the run a fresh scan pays anyway.
+ *
+ * @param {string} slicesFile Candidate dosai methods slice
+ * @param {string} projectDir Directory dosai analyses
+ * @param {Object} [options] Scan options; options.exclude prunes the walk
+ * @returns {boolean} True when the slice is newer than every source it describes
+ */
+export function dosaiSlicesFileIsFresh(slicesFile, projectDir, options = {}) {
+  if (!slicesFile || !projectDir || !safeExistsSync(slicesFile)) {
+    return false;
+  }
+  let sliceMtimeMs;
+  try {
+    sliceMtimeMs = statSync(slicesFile).mtimeMs;
+  } catch (_err) {
+    return false;
+  }
+  const excludePatterns = dosaiSourceExcludePatterns(
+    options.exclude,
+    resolve(projectDir),
+  );
+  const newest = newestDosaiSourceMtimeSync(
+    resolve(projectDir),
+    excludePatterns,
+  );
+  if (!newest.complete) {
+    return false;
+  }
+  // A directory with no file dosai analyses holds nothing that can go stale,
+  // so the slice counts as fresh.
+  return newest.mtimeMs === undefined || sliceMtimeMs >= newest.mtimeMs;
+}
+
 // The file each report returned by readDosaiReport came from, and whether it
 // was trimmed to what cdxgen reads. persistDosaiSemanticsReport copies the
 // file rather than re-serialising a trimmed or oversized report.
@@ -1978,6 +2189,23 @@ export function collectDosaiAiComponents(methodsSlice, components = []) {
           format,
         }));
       }
+      const hubReference = String(ai.Purl || "").startsWith("pkg:huggingface/")
+        ? normalizeHuggingFaceReference(ai.Purl)
+        : undefined;
+      const hubNamespace = hubReference?.repoId?.split("/")[0];
+      // A Hub model is credited to the namespace only when it is the
+      // creator's own organisation, as the source collectors do. Otherwise
+      // dosai reports the provider the code calls, which names the client
+      // rather than the model, so the creator is resolved from the model
+      // name.
+      const manufacturer = modelCreatorEntity(
+        hubNamespace
+          ? modelCreatorFromOrganization(hubNamespace)
+          : resolveModelCreator(ai.Name, {
+              fileProviders: ai.Provider ? [ai.Provider] : [],
+              provider: ai.Provider,
+            }),
+      );
       const component = {
         "bom-ref": ai.Id,
         type: "machine-learning-model",
@@ -1985,6 +2213,13 @@ export function collectDosaiAiComponents(methodsSlice, components = []) {
         version: ai.Version,
         purl: ai.Purl,
         modelCard,
+        // The creator is the manufacturer; a Hugging Face namespace that
+        // published the model is the supplier. The provider stays a
+        // property either way.
+        manufacturer,
+        supplier: hubNamespace
+          ? toHuggingFaceSupplier(hubNamespace)
+          : undefined,
         properties: [
           { name: "cdx:ai:provider", value: ai.Provider || "unknown" },
           { name: "cdx:ai:deployment", value: ai.Deployment || "unknown" },

package/lib/inventory/dosaiParsers.js

@@ -12,6 +12,7 @@ import {
 import { Purl } from "@cdxgen/cdx-purl";
 
 import { safeExistsSync } from "../core/fs.js";
+import { occurrenceLocationKey } from "./occurrenceKey.js";
 
 /**
  * Build a lowercase type/namespace/name lookup key for a purl.
@@ -68,10 +69,7 @@ export function dosaiLocation(item) {
     return undefined;
   }
   const lineNumber = location?.LineNumber || item?.LineNumber;
-  if (lineNumber && lineNumber > 0) {
-    return `${fileName}#${lineNumber}`;
-  }
-  return fileName;
+  return occurrenceLocationKey(fileName, lineNumber);
 }
 
 function dosaiSourceFileName(item) {
@@ -110,8 +108,7 @@ export function dosaiCallSiteLocation(methodCall, srcPath) {
   if (!file || lowerFile.endsWith(".dll") || lowerFile.endsWith(".exe")) {
     return undefined;
   }
-  const lineNumber = methodCall?.LineNumber;
-  return lineNumber > 0 ? `${file}#${lineNumber}` : file;
+  return occurrenceLocationKey(file, methodCall?.LineNumber);
 }
 
 function dosaiCallSiteFile(methodCall, srcPath) {