@cdxgen/cdxgen 13.5.0 → 13.7.0

package/lib/inventory/dosai.js

@@ -2,12 +2,21 @@ import { Buffer } from "node:buffer";
 import {
   closeSync,
   openSync,
+  readdirSync,
   readFileSync,
   readSync,
   realpathSync,
   statSync,
 } from "node:fs";
-import { basename, delimiter, isAbsolute, join, resolve } from "node:path";
+import {
+  basename,
+  delimiter,
+  extname,
+  isAbsolute,
+  join,
+  relative,
+  resolve,
+} from "node:path";
 import process from "node:process";
 
 import { DEBUG_MODE, readEnvironmentVariable } from "../core/activity.js";
@@ -24,12 +33,22 @@ import {
 } from "../core/fs.js";
 import { formatDuration, reapProcessTree } from "../core/processTree.js";
 import { encodeUriReference } from "../core/propertySanitizer.js";
+import {
+  normalizeHuggingFaceReference,
+  toHuggingFaceSupplier,
+} from "../parsers/huggingfaceUtils.js";
 import {
   indexJsonObject,
   MAX_JSON_TEXT_BYTES,
   readJsonMember,
   readJsonRange,
 } from "../parsers/largeJson.js";
+import {
+  modelCreatorEntity,
+  modelCreatorFromOrganization,
+  resolveModelCreator,
+} from "./aiProviders.js";
+import { isPathExcludedByGlobPatterns } from "./atomUtils.js";
 import {
   addDosaiSetValue,
   buildDosaiPurlAliasMap,
@@ -712,6 +731,198 @@ export function dosaiSectionsCdxgenReads(omittedSections) {
   );
 }
 
+// The files dosai reads from a source tree whose changes make an existing
+// methods slice stale: the C#, VB, and F# sources it analyses, the project
+// and MSBuild files that shape the analysis, and the Razor and WebForms
+// templates whose call sites it maps from build output. packages.config is
+// read by cdxgen's own dotnet analysis and holds the same package list a
+// csproj does, so a change to it can change the slice too. Solution files are
+// left out: dosai opens them only to budget its restore runs, and a changed
+// solution does not change the occurrences a slice holds.
+const DOSAI_SLICE_SOURCE_EXTENSIONS = new Set([
+  ".cs",
+  ".cshtml",
+  ".csproj",
+  ".fs",
+  ".fsi",
+  ".fsproj",
+  ".fsx",
+  ".props",
+  ".razor",
+  ".targets",
+  ".vb",
+  ".vbproj",
+]);
+const DOSAI_SLICE_SOURCE_FILES = new Set(["packages.config"]);
+
+// Directories that hold nothing dosai analyses. Its own tree walk skips the
+// build-output directories (obj for every file, bin for sources) and the
+// package and version-control directories, so a build cannot make a slice
+// stale and a dependency tree cannot slow the check down.
+const DOSAI_SLICE_SOURCE_SKIP_DIRS = new Set([
+  ".git",
+  "bin",
+  "node_modules",
+  "obj",
+]);
+
+// The tree walk is bounded so a pathological tree (a symlink farm, or entries
+// that appear faster than the walk reads them) cannot keep a scan from
+// finishing. A walk that reaches a bound leaves freshness unknown, and the
+// callers regenerate the slice, which is the run a fresh scan pays anyway.
+const DOSAI_FRESHNESS_MAX_DIRS = 50_000;
+const DOSAI_FRESHNESS_MAX_FILES = 200_000;
+
+/**
+ * The newest mtime among the source files dosai analyses under a directory.
+ *
+ * The walk is deterministic and bounded: directory entries are read and then
+ * visited depth-first in sorted name order, symlinked directories are not
+ * followed, and counts of visited directories and analysed files cap the
+ * work. Excluded paths are pruned rather than filtered, so nothing beneath an
+ * excluded directory is read.
+ *
+ * @param {string} projectDir Absolute directory dosai analyses
+ * @param {string[]} excludePatterns Exclude globs relative to projectDir
+ * @returns {{mtimeMs: number|undefined, complete: boolean}} The newest mtime seen (undefined when no analysed file was found), and whether the walk finished, which is false for an unreadable directory or file, or a tree past the bounds
+ */
+function newestDosaiSourceMtimeSync(projectDir, excludePatterns) {
+  const newest = { mtimeMs: undefined, complete: true };
+  const stack = [[projectDir, ""]];
+  let dirsVisited = 0;
+  let filesVisited = 0;
+  while (newest.complete && stack.length) {
+    if (dirsVisited++ >= DOSAI_FRESHNESS_MAX_DIRS) {
+      newest.complete = false;
+      break;
+    }
+    const [dir, relDir] = stack.pop();
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch (_err) {
+      newest.complete = false;
+      break;
+    }
+    entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    for (const entry of entries) {
+      const relPath = relDir ? `${relDir}/${entry.name}` : entry.name;
+      if (entry.isDirectory()) {
+        if (
+          !DOSAI_SLICE_SOURCE_SKIP_DIRS.has(entry.name.toLowerCase()) &&
+          !(
+            excludePatterns.length &&
+            isPathExcludedByGlobPatterns(relPath, excludePatterns)
+          )
+        ) {
+          stack.push([join(dir, entry.name), relPath]);
+        }
+        continue;
+      }
+      if (
+        !DOSAI_SLICE_SOURCE_EXTENSIONS.has(extname(entry.name).toLowerCase()) &&
+        !DOSAI_SLICE_SOURCE_FILES.has(entry.name.toLowerCase())
+      ) {
+        continue;
+      }
+      if (filesVisited++ >= DOSAI_FRESHNESS_MAX_FILES) {
+        newest.complete = false;
+        break;
+      }
+      try {
+        // statSync follows a symlink to the source it points at, which is
+        // the file dosai reads.
+        const mtimeMs = statSync(join(dir, entry.name)).mtimeMs;
+        if (newest.mtimeMs === undefined || mtimeMs > newest.mtimeMs) {
+          newest.mtimeMs = mtimeMs;
+        }
+      } catch (_err) {
+        newest.complete = false;
+        break;
+      }
+    }
+  }
+  return newest;
+}
+
+/**
+ * The cdxgen exclude patterns that apply under a directory, as globs relative
+ * to it.
+ *
+ * The patterns are matched the way the atom evidence filter matches
+ * `options.exclude`: a relative pattern matches at any depth, and an absolute
+ * pattern is anchored to the scanned directory, which for the walk means the
+ * patterns that sit beneath it are rebased to relative ones. Patterns that
+ * fall outside the directory are dropped.
+ *
+ * @param {string[]} excludes cdxgen exclude globs
+ * @param {string} projectDir Absolute directory dosai analyses
+ * @returns {string[]} Exclude globs relative to projectDir
+ */
+function dosaiSourceExcludePatterns(excludes, projectDir) {
+  const rebased = [];
+  if (!Array.isArray(excludes)) {
+    return rebased;
+  }
+  for (const pattern of excludes) {
+    if (typeof pattern !== "string" || !pattern.trim()) {
+      continue;
+    }
+    // resolve() leaves a relative pattern where it was and turns an absolute
+    // one into a path the relative() call drops the scanned directory from.
+    const underRoot = resolve(projectDir, pattern);
+    const rel = relative(projectDir, underRoot);
+    if (rel && !rel.startsWith("..") && !isAbsolute(rel)) {
+      rebased.push(rel.replaceAll("\\", "/"));
+    }
+  }
+  return rebased;
+}
+
+/**
+ * Whether an existing dosai methods slice is newer than the sources it
+ * describes.
+ *
+ * A slice written by an earlier scan holds the occurrences of the sources as
+ * they were then, so a slice that predates an edit must be regenerated rather
+ * than reused: reusing one keeps occurrences of call sites that no longer
+ * exist (issue 4675). The slice is fresh only when its mtime is not older
+ * than the newest mtime among the files dosai analyses under the directory.
+ * A missing slice, a slice whose mtime cannot be read, a source that cannot
+ * be stat-ed, and a tree that exceeds the walk bounds all count as stale: the
+ * callers regenerate the slice, which is the run a fresh scan pays anyway.
+ *
+ * @param {string} slicesFile Candidate dosai methods slice
+ * @param {string} projectDir Directory dosai analyses
+ * @param {Object} [options] Scan options; options.exclude prunes the walk
+ * @returns {boolean} True when the slice is newer than every source it describes
+ */
+export function dosaiSlicesFileIsFresh(slicesFile, projectDir, options = {}) {
+  if (!slicesFile || !projectDir || !safeExistsSync(slicesFile)) {
+    return false;
+  }
+  let sliceMtimeMs;
+  try {
+    sliceMtimeMs = statSync(slicesFile).mtimeMs;
+  } catch (_err) {
+    return false;
+  }
+  const excludePatterns = dosaiSourceExcludePatterns(
+    options.exclude,
+    resolve(projectDir),
+  );
+  const newest = newestDosaiSourceMtimeSync(
+    resolve(projectDir),
+    excludePatterns,
+  );
+  if (!newest.complete) {
+    return false;
+  }
+  // A directory with no file dosai analyses holds nothing that can go stale,
+  // so the slice counts as fresh.
+  return newest.mtimeMs === undefined || sliceMtimeMs >= newest.mtimeMs;
+}
+
 // The file each report returned by readDosaiReport came from, and whether it
 // was trimmed to what cdxgen reads. persistDosaiSemanticsReport copies the
 // file rather than re-serialising a trimmed or oversized report.
@@ -727,6 +938,105 @@ function referencedIds(entries, field) {
   return ids;
 }
 
+// The fields of a MethodCalls row cdxgen reads: the component it names (Purl
+// or Module), where the call site is (Path, FileName, LineNumber) and what it
+// calls (ClassName, CalledMethod). Everything else a report row carries -
+// arguments, evidence kinds, caller identities - no cdxgen consumer reads,
+// and a report with millions of calls cannot afford to hold them.
+const DOSAI_METHOD_CALL_FIELDS = [
+  "Purl",
+  "Module",
+  "Path",
+  "FileName",
+  "LineNumber",
+  "ClassName",
+  "CalledMethod",
+];
+
+/**
+ * The largest number of distinct method calls cdxgen keeps from one report.
+ *
+ * Reports in the millions of calls repeat the same call sites heavily, so the
+ * aggregated rows stay far below this; the cap exists so a report whose calls
+ * are all distinct degrades the occurrence evidence instead of the process.
+ *
+ * @returns {number} Cap on distinct kept rows
+ */
+export function maxDosaiMethodCalls() {
+  const configured = Number.parseInt(
+    readEnvironmentVariable("CDXGEN_DOSAI_MAX_METHOD_CALLS") || "",
+    10,
+  );
+  if (Number.isInteger(configured) && configured > 0) {
+    return configured;
+  }
+  return 1_000_000;
+}
+
+/**
+ * Build the `map` that turns report MethodCalls rows into what cdxgen keeps:
+ * rows the caller rejects are dropped, the rest are reduced to
+ * {@link DOSAI_METHOD_CALL_FIELDS} and aggregated by those fields, so a call
+ * site reached a million times is held once with a `CallCount`. Past
+ * {@link maxDosaiMethodCalls} distinct rows, further rows are dropped and one
+ * warning names the cap.
+ *
+ * @param {string} reportFile Report the rows come from, named in the warning
+ * @param {Object} [options] Options
+ * @param {Function} [options.keepMethodCall] Predicate choosing the rows to keep
+ * @param {number} [options.maxMethodCalls] Cap override, for tests
+ * @returns {{map: Function, finish: Function}} The row mapper, and the
+ *   `finish` that reports a cap reached
+ */
+function dosaiMethodCallCollector(reportFile, options = {}) {
+  const keep = options.keepMethodCall;
+  const max = options.maxMethodCalls ?? maxDosaiMethodCalls();
+  const keptByKey = new Map();
+  let dropped = 0;
+  return {
+    map(methodCall) {
+      if (keep && !keep(methodCall)) {
+        return undefined;
+      }
+      const compact = {};
+      for (const field of DOSAI_METHOD_CALL_FIELDS) {
+        if (methodCall?.[field] !== undefined) {
+          compact[field] = methodCall[field];
+        }
+      }
+      const key = DOSAI_METHOD_CALL_FIELDS.map((field) => compact[field]).join(
+        "\u0000",
+      );
+      const kept = keptByKey.get(key);
+      if (kept) {
+        kept.CallCount = (kept.CallCount ?? 1) + 1;
+        return undefined;
+      }
+      if (keptByKey.size >= max) {
+        dropped++;
+        return undefined;
+      }
+      keptByKey.set(key, compact);
+      return compact;
+    },
+    finish() {
+      if (!dropped) {
+        return;
+      }
+      console.warn(
+        `The dosai report "${reportFile}" holds more distinct method calls than cdxgen keeps (${max}); ${dropped} call sites were dropped, so occurrence evidence may be incomplete. Raise the cap with CDXGEN_DOSAI_MAX_METHOD_CALLS.`,
+      );
+      recordDegradation("dosai.method-calls-capped", {
+        ecosystem: "csharp",
+        tool: "dosai",
+        impact: "components",
+        command: reportFile,
+        detail: `The dosai report holds more distinct method calls than the cap of ${max}, so ${dropped} call sites were dropped from the occurrence evidence.`,
+      });
+    },
+  };
+}
+
 function readDosaiReport(reportFile, options, readTrimmed) {
   if (!reportFile || !safeExistsSync(reportFile)) {
     return undefined;
@@ -761,7 +1071,13 @@ function readDosaiReport(reportFile, options, readTrimmed) {
     }
     return report;
   } catch (err) {
-    if (DEBUG_MODE) {
+    if (err instanceof RangeError) {
+      // A heap or string limit is a report-size problem the user can act on,
+      // unlike a malformed report, so it is named even without debug output.
+      console.warn(
+        `The dosai report "${reportFile}" is too large to read: ${err.message}. Raise the Node.js heap, for example NODE_OPTIONS="--max-old-space-size=8192", or write the report with dosai's --sections so it holds less.`,
+      );
+    } else if (DEBUG_MODE) {
       console.log(
         `Unable to read the dosai report "${reportFile}": ${err.message}`,
       );
@@ -776,25 +1092,34 @@ function readDosaiReport(reportFile, options, readTrimmed) {
  * A report that fits in one JavaScript string is parsed whole. A larger one is
  * read in bounded runs, keeping the requested sections, the CallGraph nodes and
  * edges that PackageReachability references, and the MethodCalls accepted by
- * `keepMethodCall` (none when it is omitted). Nothing cdxgen reads is lost.
+ * `keepMethodCall` (none when it is omitted). A kept MethodCalls row is
+ * reduced to the fields cdxgen reads and aggregated by them, so a call site
+ * reached many times is held once with a `CallCount`; distinct rows past
+ * {@link maxDosaiMethodCalls} are dropped with one warning. Nothing cdxgen
+ * reads is lost.
  *
  * @param {string} reportFile Path to the dosai methods JSON
  * @param {Object} [options] Options
  * @param {string[]} [options.sections] Sections to keep from a large report (default: every section cdxgen reads)
  * @param {Function} [options.keepMethodCall] Predicate choosing the MethodCalls to keep from a large report
+ * @param {number} [options.maxMethodCalls] Cap on distinct kept MethodCalls, for tests
  * @param {number} [options.maxTextBytes] Largest report parsed whole, for tests
  * @returns {Object|undefined} Parsed report, or undefined when missing or invalid
  */
 export function readDosaiMethodsReport(reportFile, options = {}) {
   const sections = options.sections || DOSAI_METHODS_SECTIONS;
-  return readDosaiReport(reportFile, options, (report, read) => {
+  const collector =
+    sections.includes("MethodCalls") && options.keepMethodCall
+      ? dosaiMethodCallCollector(reportFile, options)
+      : undefined;
+  const report = readDosaiReport(reportFile, options, (report, read) => {
     for (const section of sections) {
       if (section !== "CallGraph" && section !== "MethodCalls") {
         read(section);
       }
     }
-    if (sections.includes("MethodCalls") && options.keepMethodCall) {
-      read("MethodCalls", { filter: options.keepMethodCall });
+    if (collector) {
+      read("MethodCalls", { map: collector.map });
     }
     if (sections.includes("CallGraph")) {
       // CallGraph precedes PackageReachability in the file, which is why the
@@ -809,6 +1134,26 @@ export function readDosaiMethodsReport(reportFile, options = {}) {
       });
     }
   });
+  // A report parsed whole kept every row, as it always has: reduce them to
+  // the fields cdxgen reads and aggregate repeats, without the caller's
+  // predicate, which the whole parse has never applied. A report that took
+  // the trimmed path already went through the collector above.
+  if (report && !dosaiReportSources.get(report)?.trimmed) {
+    const wholeCollector = dosaiMethodCallCollector(reportFile, {
+      maxMethodCalls: options.maxMethodCalls,
+    });
+    const reduced = [];
+    for (const methodCall of report.MethodCalls || []) {
+      const kept = wholeCollector.map(methodCall);
+      if (kept !== undefined) {
+        reduced.push(kept);
+      }
+    }
+    report.MethodCalls = reduced;
+    wholeCollector.finish();
+  }
+  collector?.finish();
+  return report;
 }
 
 /**
@@ -1844,6 +2189,23 @@ export function collectDosaiAiComponents(methodsSlice, components = []) {
           format,
         }));
       }
+      const hubReference = String(ai.Purl || "").startsWith("pkg:huggingface/")
+        ? normalizeHuggingFaceReference(ai.Purl)
+        : undefined;
+      const hubNamespace = hubReference?.repoId?.split("/")[0];
+      // A Hub model is credited to the namespace only when it is the
+      // creator's own organisation, as the source collectors do. Otherwise
+      // dosai reports the provider the code calls, which names the client
+      // rather than the model, so the creator is resolved from the model
+      // name.
+      const manufacturer = modelCreatorEntity(
+        hubNamespace
+          ? modelCreatorFromOrganization(hubNamespace)
+          : resolveModelCreator(ai.Name, {
+              fileProviders: ai.Provider ? [ai.Provider] : [],
+              provider: ai.Provider,
+            }),
+      );
       const component = {
         "bom-ref": ai.Id,
         type: "machine-learning-model",
@@ -1851,6 +2213,13 @@ export function collectDosaiAiComponents(methodsSlice, components = []) {
         version: ai.Version,
         purl: ai.Purl,
         modelCard,
+        // The creator is the manufacturer; a Hugging Face namespace that
+        // published the model is the supplier. The provider stays a
+        // property either way.
+        manufacturer,
+        supplier: hubNamespace
+          ? toHuggingFaceSupplier(hubNamespace)
+          : undefined,
         properties: [
           { name: "cdx:ai:provider", value: ai.Provider || "unknown" },
           { name: "cdx:ai:deployment", value: ai.Deployment || "unknown" },

package/lib/inventory/dosaiParsers.js

@@ -12,6 +12,7 @@ import {
 import { Purl } from "@cdxgen/cdx-purl";
 
 import { safeExistsSync } from "../core/fs.js";
+import { occurrenceLocationKey } from "./occurrenceKey.js";
 
 /**
  * Build a lowercase type/namespace/name lookup key for a purl.
@@ -68,10 +69,7 @@ export function dosaiLocation(item) {
     return undefined;
   }
   const lineNumber = location?.LineNumber || item?.LineNumber;
-  if (lineNumber && lineNumber > 0) {
-    return `${fileName}#${lineNumber}`;
-  }
-  return fileName;
+  return occurrenceLocationKey(fileName, lineNumber);
 }
 
 function dosaiSourceFileName(item) {
@@ -110,8 +108,7 @@ export function dosaiCallSiteLocation(methodCall, srcPath) {
   if (!file || lowerFile.endsWith(".dll") || lowerFile.endsWith(".exe")) {
     return undefined;
   }
-  const lineNumber = methodCall?.LineNumber;
-  return lineNumber > 0 ? `${file}#${lineNumber}` : file;
+  return occurrenceLocationKey(file, methodCall?.LineNumber);
 }
 
 function dosaiCallSiteFile(methodCall, srcPath) {